From 09c820a7e5c102066f368b84dbbc831b4b54741d Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Thu, 1 Aug 2019 21:37:41 +0800 Subject: [PATCH 1/3] [AutoTVM] Fix hang/crash issues on feature extraction --- .../tvm/autotvm/tuner/xgboost_cost_model.py | 127 +++++++++++------- src/autotvm/touch_extractor.cc | 4 +- 2 files changed, 78 insertions(+), 53 deletions(-) diff --git a/python/tvm/autotvm/tuner/xgboost_cost_model.py b/python/tvm/autotvm/tuner/xgboost_cost_model.py index e278957c8af3..967877efbfad 100644 --- a/python/tvm/autotvm/tuner/xgboost_cost_model.py +++ b/python/tvm/autotvm/tuner/xgboost_cost_model.py @@ -312,9 +312,15 @@ def _get_feature(self, indexes): for i, fea in zip(need_extract, feas): fea_cache[i] = fea - ret = np.empty((len(indexes), fea_cache[indexes[0]].shape[-1]), dtype=np.float32) + feature_len = None + for idx in indexes: + if fea_cache[idx] is not None: + feature_len = fea_cache[idx].shape[-1] + + ret = np.empty((len(indexes), feature_len), dtype=np.float32) for i, ii in enumerate(indexes): - ret[i, :] = fea_cache[ii] + t = fea_cache[ii] + ret[i, :] = t if t is not None else 0 return ret def __del__(self): @@ -327,71 +333,88 @@ def __del__(self): def _extract_itervar_feature_index(index): """extract iteration var feature for an index in extract_space""" - config = _extract_space.get(index) - with _extract_target: - sch, args = _extract_task.instantiate(config) - fea = feature.get_itervar_feature_flatten(sch, args, take_log=True) - fea = np.concatenate((fea, list(config.get_other_option().values()))) - return fea + try: + config = _extract_space.get(index) + with _extract_target: + sch, args = _extract_task.instantiate(config) + fea = feature.get_itervar_feature_flatten(sch, args, take_log=True) + fea = np.concatenate((fea, list(config.get_other_option().values()))) + return fea + except Exception: # pylint: disable=broad-except + return None def _extract_itervar_feature_log(arg): """extract iteration var feature for log items""" - inp, res = arg - config = inp.config - with inp.target: - sch, args = inp.task.instantiate(config) - fea = feature.get_itervar_feature_flatten(sch, args, take_log=True) - x = np.concatenate((fea, list(config.get_other_option().values()))) - - if res.error_no == 0: - y = inp.task.flop / np.mean(res.costs) - else: - y = 0.0 - return x, y + try: + inp, res = arg + config = inp.config + with inp.target: + sch, args = inp.task.instantiate(config) + fea = feature.get_itervar_feature_flatten(sch, args, take_log=True) + x = np.concatenate((fea, list(config.get_other_option().values()))) + + if res.error_no == 0: + y = inp.task.flop / np.mean(res.costs) + else: + y = 0.0 + return x, y + except Exception: # pylint: disable=broad-except + return None def _extract_knob_feature_index(index): """extract knob feature for an index in extract_space""" - config = _extract_space.get(index) - return config.get_flatten_feature() + try: + config = _extract_space.get(index) + return config.get_flatten_feature() + except Exception: + return None def _extract_knob_feature_log(arg): """extract knob feature for log items""" - inp, res = arg - config = inp.config - x = config.get_flatten_feature() - - if res.error_no == 0: - with inp.target: # necessary, for calculating flops of this task - inp.task.instantiate(config) - y = inp.task.flop / np.mean(res.costs) - else: - y = 0.0 - return x, y + try: + inp, res = arg + config = inp.config + x = config.get_flatten_feature() + + if res.error_no == 0: + with inp.target: # necessary, for calculating flops of this task + inp.task.instantiate(config) + y = inp.task.flop / np.mean(res.costs) + else: + y = 0.0 + return x, y + except Exception: # pylint: disable=broad-except + return None def _extract_curve_feature_index(index): """extract sampled curve feature for an index in extract_space""" - config = _extract_space.get(index) - with _extract_target: - sch, args = _extract_task.instantiate(config) - fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20) - fea = np.concatenate((fea, list(config.get_other_option().values()))) - return np.array(fea) + try: + config = _extract_space.get(index) + with _extract_target: + sch, args = _extract_task.instantiate(config) + fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20) + fea = np.concatenate((fea, list(config.get_other_option().values()))) + return np.array(fea) + except Exception: # pylint: disable=broad-except + return None def _extract_curve_feature_log(arg): """extract sampled curve feature for log items""" - inp, res = arg - config = inp.config - with inp.target: - sch, args = inp.task.instantiate(config) - fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20) - x = np.concatenate((fea, list(config.get_other_option().values()))) - - if res.error_no == 0: - y = inp.task.flop / np.mean(res.costs) - else: - y = 0.0 - return x, y - + try: + inp, res = arg + config = inp.config + with inp.target: + sch, args = inp.task.instantiate(config) + fea = feature.get_buffer_curve_sample_flatten(sch, args, sample_n=20) + x = np.concatenate((fea, list(config.get_other_option().values()))) + + if res.error_no == 0: + y = inp.task.flop / np.mean(res.costs) + else: + y = 0.0 + return x, y + except Exception: # pylint: disable=broad-except + return None def custom_callback(stopping_rounds, metric, fevals, evals=(), log_file=None, maximize=False, verbose_eval=True): diff --git a/src/autotvm/touch_extractor.cc b/src/autotvm/touch_extractor.cc index 002b970588ec..d40ebd68676b 100644 --- a/src/autotvm/touch_extractor.cc +++ b/src/autotvm/touch_extractor.cc @@ -131,7 +131,9 @@ void TouchExtractor::ExitItervar_() { } itervar_stack_.pop_back(); - topdown_product_ /= itervar_map[var].length; + int64_t length = itervar_map[var].length; + if (length != 0) + topdown_product_ /= length; int64_t bottomup_product = -1; for (auto kv : itervar_map[var].touch_feature) { bottomup_product = std::max(bottomup_product, kv.second.count * kv.second.reuse); From 5c7f0a506b3b9adc318ef00b1916c836e7ad25b2 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 2 Aug 2019 14:27:21 +0800 Subject: [PATCH 2/3] Update xgboost_cost_model.py --- python/tvm/autotvm/tuner/xgboost_cost_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tvm/autotvm/tuner/xgboost_cost_model.py b/python/tvm/autotvm/tuner/xgboost_cost_model.py index 967877efbfad..2060dbd8c4cf 100644 --- a/python/tvm/autotvm/tuner/xgboost_cost_model.py +++ b/python/tvm/autotvm/tuner/xgboost_cost_model.py @@ -316,6 +316,7 @@ def _get_feature(self, indexes): for idx in indexes: if fea_cache[idx] is not None: feature_len = fea_cache[idx].shape[-1] + break ret = np.empty((len(indexes), feature_len), dtype=np.float32) for i, ii in enumerate(indexes): From 542f9854f9b5650034a0103c83bfa4cb86ad11a0 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Fri, 2 Aug 2019 22:01:17 +0800 Subject: [PATCH 3/3] fix lint --- python/tvm/autotvm/tuner/xgboost_cost_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tvm/autotvm/tuner/xgboost_cost_model.py b/python/tvm/autotvm/tuner/xgboost_cost_model.py index 2060dbd8c4cf..265365144639 100644 --- a/python/tvm/autotvm/tuner/xgboost_cost_model.py +++ b/python/tvm/autotvm/tuner/xgboost_cost_model.py @@ -367,7 +367,7 @@ def _extract_knob_feature_index(index): try: config = _extract_space.get(index) return config.get_flatten_feature() - except Exception: + except Exception: # pylint: disable=broad-except return None def _extract_knob_feature_log(arg):