From f512317fe40d954052c87d6f42d82d6f6c90f69c Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 13 Apr 2021 17:47:38 -0700 Subject: [PATCH 1/4] e-notation for large floats --- deepspeed/runtime/config.py | 3 ++- deepspeed/runtime/config_utils.py | 40 +++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/deepspeed/runtime/config.py b/deepspeed/runtime/config.py index 11e1d4037c8e..b9ba6cf37cba 100755 --- a/deepspeed/runtime/config.py +++ b/deepspeed/runtime/config.py @@ -9,7 +9,7 @@ from .constants import * from .fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, DELAYED_SHIFT, MIN_LOSS_SCALE -from .config_utils import get_scalar_param, dict_raise_error_on_duplicate_keys +from .config_utils import get_scalar_param, dict_raise_error_on_duplicate_keys, ScientificNotationEncoder from .zero.config import DeepSpeedZeroConfig from .zero.constants import * from .activation_checkpointing.config import DeepSpeedActivationCheckpointingConfig @@ -744,6 +744,7 @@ def print(self, name): json.dumps(self._param_dict, sort_keys=True, indent=4, + cls=ScientificNotationEncoder, separators=(',', ':')))) diff --git a/deepspeed/runtime/config_utils.py b/deepspeed/runtime/config_utils.py index 62782852a3d2..72d5f7a02722 100755 --- a/deepspeed/runtime/config_utils.py +++ b/deepspeed/runtime/config_utils.py @@ -6,7 +6,38 @@ Collection of DeepSpeed configuration utilities """ import json -from collections import Counter +from collections import Counter, Mapping, Sequence + + +# adapted from https://stackoverflow.com/a/50701137/9201239 +class ScientificNotationEncoder(json.JSONEncoder): + """ + This class overrides ``json.dumps`` default formatter. + + This version keeps everything as normal except formats floats bigger than 1e3 using scientific notation. + + Just pass ``cls=ScientificNotationEncoder`` to ``json.dumps`` to activate it + + """ + def iterencode(self, o, _one_shot=False, level=0): + indent = self.indent if self.indent is not None else 4 + prefix_close = " " * level * indent + level += 1 + prefix = " " * level * indent + if isinstance(o, float): + if o > 1e3: + return f"{o:e}" + else: + return f"{o}" + elif isinstance(o, Mapping): + x = [ + f'\n{prefix}"{k}": {self.iterencode(v, level=level)}' for k, + v in o.items() + ] + return "{" + ', '.join(x) + f"\n{prefix_close}}}" + elif isinstance(o, Sequence) and not isinstance(o, str): + return f"[{ f', '.join(map(self.iterencode, o)) }]" + return "\n, ".join(super().iterencode(o, _one_shot)) class DeepSpeedConfigObject(object): @@ -17,7 +48,12 @@ def repr(self): return self.__dict__ def __repr__(self): - return json.dumps(self.__dict__, sort_keys=True, indent=4) + return json.dumps( + self.__dict__, + sort_keys=True, + indent=4, + cls=ScientificNotationEncoder, + ) def get_scalar_param(param_dict, param_name, param_default_value): From 7ec68c0ab8a929216c78af08d97bad988de0a3a5 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 13 Apr 2021 17:52:21 -0700 Subject: [PATCH 2/4] handle ints too --- deepspeed/runtime/config_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepspeed/runtime/config_utils.py b/deepspeed/runtime/config_utils.py index 72d5f7a02722..ed4fabd3cd42 100755 --- a/deepspeed/runtime/config_utils.py +++ b/deepspeed/runtime/config_utils.py @@ -14,7 +14,7 @@ class ScientificNotationEncoder(json.JSONEncoder): """ This class overrides ``json.dumps`` default formatter. - This version keeps everything as normal except formats floats bigger than 1e3 using scientific notation. + This version keeps everything as normal except formats numbers bigger than 1e3 using scientific notation. Just pass ``cls=ScientificNotationEncoder`` to ``json.dumps`` to activate it @@ -24,7 +24,7 @@ def iterencode(self, o, _one_shot=False, level=0): prefix_close = " " * level * indent level += 1 prefix = " " * level * indent - if isinstance(o, float): + if isinstance(o, float) or isinstance(o, int): if o > 1e3: return f"{o:e}" else: From c8476fbd4372f8f4dbcd8346d119860e4617fae2 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 13 Apr 2021 17:56:34 -0700 Subject: [PATCH 3/4] readability --- deepspeed/runtime/config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/runtime/config_utils.py b/deepspeed/runtime/config_utils.py index ed4fabd3cd42..d2b05dbfea95 100755 --- a/deepspeed/runtime/config_utils.py +++ b/deepspeed/runtime/config_utils.py @@ -34,7 +34,7 @@ def iterencode(self, o, _one_shot=False, level=0): f'\n{prefix}"{k}": {self.iterencode(v, level=level)}' for k, v in o.items() ] - return "{" + ', '.join(x) + f"\n{prefix_close}}}" + return "{" + ', '.join(x) + f"\n{prefix_close}" + "}" elif isinstance(o, Sequence) and not isinstance(o, str): return f"[{ f', '.join(map(self.iterencode, o)) }]" return "\n, ".join(super().iterencode(o, _one_shot)) From 5c3bdb1aa4ba77245aedb853f023fea2b16504fd Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 13 Apr 2021 18:11:55 -0700 Subject: [PATCH 4/4] handle bool --- deepspeed/runtime/config_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepspeed/runtime/config_utils.py b/deepspeed/runtime/config_utils.py index d2b05dbfea95..12711d56f7f6 100755 --- a/deepspeed/runtime/config_utils.py +++ b/deepspeed/runtime/config_utils.py @@ -24,7 +24,9 @@ def iterencode(self, o, _one_shot=False, level=0): prefix_close = " " * level * indent level += 1 prefix = " " * level * indent - if isinstance(o, float) or isinstance(o, int): + if isinstance(o, bool): + return "true" if o else "false" + elif isinstance(o, float) or isinstance(o, int): if o > 1e3: return f"{o:e}" else: