From 8c34d3d9a7b8e766d478065fefda765520222e04 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 15 Sep 2020 22:06:13 +0000 Subject: [PATCH 1/2] initial commit --- python/mxnet/gluon/block.py | 42 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 8fd7dd318758..40504e433b1d 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -21,7 +21,6 @@ __all__ = ['Block', 'HybridBlock', 'SymbolBlock'] import copy -import inspect import warnings import weakref from collections import OrderedDict, defaultdict @@ -948,8 +947,8 @@ def _get_graph_v1(self, *args): flatten_inputs.append(None) grouped_inputs = _regroup(flatten_inputs, self._in_format) + params = {i: j.var() for i, j in self._reg_params.items()} with _block_scope(self): - params = {i: j.var() for i, j in self._reg_params.items()} out = self.hybrid_forward(symbol, *grouped_inputs, **params) # pylint: disable=no-value-for-parameter out, self._out_format = _flatten(out, "output") @@ -985,7 +984,7 @@ def _get_graph_v2(self, *args): def _get_graph(self, *args): if not self._cached_graph: - if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: + if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: # Gluon 1 return self._get_graph_v1(*args) else: # Gluon 2 based on deferred compute mode return self._get_graph_v2(*args) @@ -1133,7 +1132,7 @@ def _call_cached_op(self, *args): out = [out] return _regroup(out, self._out_format) - def optimize_for(self, x, *args, backend=None, backend_opts=None, clear=True, **kwargs): + def optimize_for(self, x, *args, backend=None, backend_opts=None, **kwargs): """Partitions the current HybridBlock and optimizes it for a given backend without executing a forward pass. Modifies the HybridBlock in-place. @@ -1163,7 +1162,6 @@ def optimize_for(self, x, *args, backend=None, backend_opts=None, clear=True, ** The name of backend, as registered in `SubgraphBackendRegistry`, default None backend_opts : dict of user-specified options to pass to the backend for partitioning, optional Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty` - clear : clears any previous optimizations static_alloc : bool, default False Statically allocate memory to improve speed. Memory usage may increase. static_shape : bool, default False @@ -1173,7 +1171,7 @@ def optimize_for(self, x, *args, backend=None, backend_opts=None, clear=True, ** """ # do hybrize API call - self.hybridize(True, backend, backend_opts, clear, **kwargs) + self.hybridize(True, backend, backend_opts, **kwargs) # do part of forward API call has_symbol, has_ndarray, ctx_set, _ = _gather_type_ctx_info([x] + list(args)) @@ -1215,7 +1213,7 @@ def register_child(self, block, name=None): self._active = False self._clear_cached_op() - def hybridize(self, active=True, backend=None, backend_opts=None, clear=True, **kwargs): + def hybridize(self, active=True, backend=None, backend_opts=None, **kwargs): """Activates or deactivates :py:class:`HybridBlock` s recursively. Has no effect on non-hybrid children. @@ -1227,7 +1225,6 @@ def hybridize(self, active=True, backend=None, backend_opts=None, clear=True, ** The name of backend, as registered in `SubgraphBackendRegistry`, default None backend_opts : dict of user-specified options to pass to the backend for partitioning, optional Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty` - clear : clears any previous optimizations static_alloc : bool, default False Statically allocate memory to improve speed. Memory usage may increase. static_shape : bool, default False @@ -1244,8 +1241,7 @@ def hybridize(self, active=True, backend=None, backend_opts=None, clear=True, ** self._active = active self._flags = list(kwargs.items()) - if clear: - self._clear_cached_op() + self._clear_cached_op() if active and self._forward_hooks or self._forward_pre_hooks: warnings.warn('"{block}" is being hybridized while still having forward hook/pre-hook. ' 'If "{block}" is a child of HybridBlock, the hooks will not take effect.' @@ -1278,7 +1274,7 @@ def _infer_attrs(self, infer_fn, attr, *args): def infer_shape(self, *args): """Infers shape of Parameters from inputs.""" - if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: + if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: # Gluon 1 based on F: hybrid_forward is defined by user self._infer_attrs('infer_shape', 'shape', *args) else: @@ -1349,11 +1345,8 @@ def export(self, path, epoch=0, remove_amp_cast=True): if name in arg_names: arg_dict['arg:{}'.format(name)] = param._reduce() else: - if name not in aux_names: - warnings.warn('Parameter "{name}" is not found in the graph. ' - .format(name=name), stacklevel=3) - else: - arg_dict['aux:%s'%name] = param._reduce() + assert name in aux_names + arg_dict['aux:{}'.format(name)] = param._reduce() save_fn = _mx_npx.save if is_np_array() else ndarray.save params_filename = '%s-%04d.params'%(path, epoch) save_fn(params_filename, arg_dict) @@ -1389,7 +1382,7 @@ def c_callback(name, op_name, array): cld()._monitor_all = monitor_all def __call__(self, x, *args): - if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: + if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: # Gluon 1 based on F: hybrid_forward is defined by user return super().__call__(x, *args) else: # Gluon 2 based on deferred compute mode @@ -1412,8 +1405,7 @@ def __call__(self, x, *args): # HybridBlock is a child block of a HybridBlock that has been hybridized. return super().__call__(x, *args) - with x.ctx: - return self._call_cached_op(x, *args) + return self._call_cached_op(x, *args) def forward(self, x, *args): """Defines the forward computation. Arguments can be either @@ -1449,8 +1441,8 @@ def forward(self, x, *args): return self.hybrid_forward(ndarray, x, *args, **params) + params = {i: j.var() for i, j in self._reg_params.items()} with _block_scope(self): - params = {i: j.var() for i, j in self._reg_params.items()} return self.hybrid_forward(symbol, x, *args, **params) def hybrid_forward(self, F, x, *args, **kwargs): @@ -1516,7 +1508,8 @@ class SymbolBlock(HybridBlock): >>> print(feat_model(x)) """ @staticmethod - def imports(symbol_file, input_names, param_file=None, ctx=None): + def imports(symbol_file, input_names, param_file=None, ctx=None, allow_missing=False, + ignore_extra=False): """Import model previously saved by `gluon.HybridBlock.export` as a `gluon.SymbolBlock` for use in Gluon. @@ -1530,6 +1523,11 @@ def imports(symbol_file, input_names, param_file=None, ctx=None): Path to parameter file. ctx : Context, default None The context to initialize `gluon.SymbolBlock` on. + allow_missing : bool, default False + Whether to silently skip loading parameters not represents in the file. + ignore_extra : bool, default False + Whether to silently ignore parameters from the file that are not + present in this Block. Returns ------- @@ -1562,7 +1560,7 @@ def imports(symbol_file, input_names, param_file=None, ctx=None): inputs = [symbol.var(i).as_np_ndarray() if is_np_array() else symbol.var(i) for i in input_names] ret = SymbolBlock(sym, inputs) if param_file is not None: - ret.load_parameters(param_file, ctx=ctx, cast_dtype=True, dtype_source='saved') + ret.load_parameters(param_file, ctx, allow_missing, ignore_extra, True, 'saved') return ret def __repr__(self): From 077329570ffe3d8def32a20f24fe30b06967ee14 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 16 Sep 2020 22:22:21 +0000 Subject: [PATCH 2/2] fixed merge issues --- python/mxnet/gluon/block.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 40504e433b1d..d430aee90b4e 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -21,6 +21,7 @@ __all__ = ['Block', 'HybridBlock', 'SymbolBlock'] import copy +import inspect import warnings import weakref from collections import OrderedDict, defaultdict @@ -947,8 +948,8 @@ def _get_graph_v1(self, *args): flatten_inputs.append(None) grouped_inputs = _regroup(flatten_inputs, self._in_format) - params = {i: j.var() for i, j in self._reg_params.items()} with _block_scope(self): + params = {i: j.var() for i, j in self._reg_params.items()} out = self.hybrid_forward(symbol, *grouped_inputs, **params) # pylint: disable=no-value-for-parameter out, self._out_format = _flatten(out, "output") @@ -984,7 +985,7 @@ def _get_graph_v2(self, *args): def _get_graph(self, *args): if not self._cached_graph: - if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: # Gluon 1 + if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: return self._get_graph_v1(*args) else: # Gluon 2 based on deferred compute mode return self._get_graph_v2(*args) @@ -1132,7 +1133,7 @@ def _call_cached_op(self, *args): out = [out] return _regroup(out, self._out_format) - def optimize_for(self, x, *args, backend=None, backend_opts=None, **kwargs): + def optimize_for(self, x, *args, backend=None, backend_opts=None, clear=True, **kwargs): """Partitions the current HybridBlock and optimizes it for a given backend without executing a forward pass. Modifies the HybridBlock in-place. @@ -1162,6 +1163,7 @@ def optimize_for(self, x, *args, backend=None, backend_opts=None, **kwargs): The name of backend, as registered in `SubgraphBackendRegistry`, default None backend_opts : dict of user-specified options to pass to the backend for partitioning, optional Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty` + clear : clears any previous optimizations static_alloc : bool, default False Statically allocate memory to improve speed. Memory usage may increase. static_shape : bool, default False @@ -1171,7 +1173,7 @@ def optimize_for(self, x, *args, backend=None, backend_opts=None, **kwargs): """ # do hybrize API call - self.hybridize(True, backend, backend_opts, **kwargs) + self.hybridize(True, backend, backend_opts, clear, **kwargs) # do part of forward API call has_symbol, has_ndarray, ctx_set, _ = _gather_type_ctx_info([x] + list(args)) @@ -1213,7 +1215,7 @@ def register_child(self, block, name=None): self._active = False self._clear_cached_op() - def hybridize(self, active=True, backend=None, backend_opts=None, **kwargs): + def hybridize(self, active=True, backend=None, backend_opts=None, clear=True, **kwargs): """Activates or deactivates :py:class:`HybridBlock` s recursively. Has no effect on non-hybrid children. @@ -1225,6 +1227,7 @@ def hybridize(self, active=True, backend=None, backend_opts=None, **kwargs): The name of backend, as registered in `SubgraphBackendRegistry`, default None backend_opts : dict of user-specified options to pass to the backend for partitioning, optional Passed on to `PrePartition` and `PostPartition` functions of `SubgraphProperty` + clear : clears any previous optimizations static_alloc : bool, default False Statically allocate memory to improve speed. Memory usage may increase. static_shape : bool, default False @@ -1241,7 +1244,8 @@ def hybridize(self, active=True, backend=None, backend_opts=None, **kwargs): self._active = active self._flags = list(kwargs.items()) - self._clear_cached_op() + if clear: + self._clear_cached_op() if active and self._forward_hooks or self._forward_pre_hooks: warnings.warn('"{block}" is being hybridized while still having forward hook/pre-hook. ' 'If "{block}" is a child of HybridBlock, the hooks will not take effect.' @@ -1274,7 +1278,7 @@ def _infer_attrs(self, infer_fn, attr, *args): def infer_shape(self, *args): """Infers shape of Parameters from inputs.""" - if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: + if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: # Gluon 1 based on F: hybrid_forward is defined by user self._infer_attrs('infer_shape', 'shape', *args) else: @@ -1345,8 +1349,11 @@ def export(self, path, epoch=0, remove_amp_cast=True): if name in arg_names: arg_dict['arg:{}'.format(name)] = param._reduce() else: - assert name in aux_names - arg_dict['aux:{}'.format(name)] = param._reduce() + if name not in aux_names: + warnings.warn('Parameter "{name}" is not found in the graph. ' + .format(name=name), stacklevel=3) + else: + arg_dict['aux:%s'%name] = param._reduce() save_fn = _mx_npx.save if is_np_array() else ndarray.save params_filename = '%s-%04d.params'%(path, epoch) save_fn(params_filename, arg_dict) @@ -1382,7 +1389,7 @@ def c_callback(name, op_name, array): cld()._monitor_all = monitor_all def __call__(self, x, *args): - if self.hybrid_forward.__func__ is not HybridBlock.hybrid_forward: + if inspect.unwrap(self.hybrid_forward.__func__) is not HybridBlock.hybrid_forward: # Gluon 1 based on F: hybrid_forward is defined by user return super().__call__(x, *args) else: # Gluon 2 based on deferred compute mode @@ -1405,7 +1412,8 @@ def __call__(self, x, *args): # HybridBlock is a child block of a HybridBlock that has been hybridized. return super().__call__(x, *args) - return self._call_cached_op(x, *args) + with x.ctx: + return self._call_cached_op(x, *args) def forward(self, x, *args): """Defines the forward computation. Arguments can be either @@ -1441,8 +1449,8 @@ def forward(self, x, *args): return self.hybrid_forward(ndarray, x, *args, **params) - params = {i: j.var() for i, j in self._reg_params.items()} with _block_scope(self): + params = {i: j.var() for i, j in self._reg_params.items()} return self.hybrid_forward(symbol, x, *args, **params) def hybrid_forward(self, F, x, *args, **kwargs):