From 8820aaecf33678436c10b9201c4decc7cb57cbcc Mon Sep 17 00:00:00 2001 From: LeiWang1999 Date: Thu, 13 Feb 2025 09:32:32 +0000 Subject: [PATCH] Remove attrs dependency and migrate to dataclasses - Remove attrs from base requirements and constraints - Replace attr.s decorator with @dataclass in memory_plan.py - Simplify Region class definition using dataclass Remove attrs dependency from installation scripts and documentation - Remove attrs from conda recipe requirements - Remove attrs from Docker installation scripts for Ubuntu - Update documentation to remove attrs from pip install instructions lint fix Remove decorator dependency from various installation scripts and files This commit removes the decorator package from: - Conda recipe requirements - Docker installation scripts - Python requirements generation - Documentation installation instructions Additionally, it introduces a local implementation of the decorator module in the TVM project to maintain existing functionality without external dependency. Refactor decorator module to remove external dependency This commit completes the local implementation of the decorator module by: - Cleaning up license headers - Formatting code to improve readability - Preserving the full functionality of the original decorator library - Ensuring consistent code style with the rest of the TVM project Refactor decorator module: Code formatting and style improvements This commit continues the work of refactoring the decorator module by: - Applying consistent code formatting - Using double quotes for string literals - Improving code readability through whitespace and line breaks - Maintaining the module's existing functionality Disable all pylint checks in decorator module Modify pylint configuration to disable all checks for the decorator module, simplifying linting rules while maintaining the module's functionality. Add decorator.py to rat-excludes for package source tracking Exclude the downloaded package source file for the decorator module from license header checks, maintaining consistency with previous refactoring efforts around the decorator module. Update LICENSE and NOTICE for decorator module Add attribution and licensing details for the decorator module in LICENSE and NOTICE files. Update rat-excludes to reflect the package source tracking for the decorator module. --- LICENSE | 1 + NOTICE | 16 + conda/recipe/meta.yaml | 1 - docker/Dockerfile.demo_opencl | 2 +- .../ubuntu2004_install_python_package.sh | 1 - .../install/ubuntu_install_python_package.sh | 1 - docs/install/from_source.rst | 2 +- python/gen_requirements.py | 2 - python/tvm/_ffi/base.py | 4 +- python/tvm/_ffi/decorator.py | 467 ++++++++++++++++++ tests/lint/rat-excludes | 3 + 11 files changed, 491 insertions(+), 9 deletions(-) create mode 100644 python/tvm/_ffi/decorator.py diff --git a/LICENSE b/LICENSE index f6126d011a9a..66fb21c34ee0 100644 --- a/LICENSE +++ b/LICENSE @@ -222,6 +222,7 @@ BSD 2-clause License 3rdparty/picojson 3rdparty/dmlc-core/include/dmlc/concurrentqueue.h +python/tvm/_ffi/decorator.py BSD 2-clause License + zlib License diff --git a/NOTICE b/NOTICE index 35d370988822..72da4ed6f373 100644 --- a/NOTICE +++ b/NOTICE @@ -3,3 +3,19 @@ Copyright 2019-2023 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the decorator project +(https://github.com/micheles/decorator) +Copyright (c) 2005-2021, Michele Simionato +Licensed under the BSD license. + +The text of the BSD license: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in bytecode form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index e7294b71b2cd..9aa0746a30fe 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -82,7 +82,6 @@ outputs: - {{ pin_subpackage(pkg_name + '-libs', exact=True) }} run: - python - - decorator - psutil - scipy - typing_extensions diff --git a/docker/Dockerfile.demo_opencl b/docker/Dockerfile.demo_opencl index 52995496fdfe..ed83f66d98af 100644 --- a/docker/Dockerfile.demo_opencl +++ b/docker/Dockerfile.demo_opencl @@ -40,7 +40,7 @@ RUN apt-install-and-clear -y apt-utils sudo cmake g++ llvm git libopenblas-dev RUN echo "Installing Python" RUN apt-install-and-clear -y python3-dev python3-pip -RUN pip3 install setuptools numpy pytest cython decorator scipy tornado psutil xgboost +RUN pip3 install setuptools numpy pytest cython scipy tornado psutil xgboost RUN echo "Installing Jupyter notebook" RUN pip3 install matplotlib Image "Pillow<7" jupyter[notebook] diff --git a/docker/install/ubuntu2004_install_python_package.sh b/docker/install/ubuntu2004_install_python_package.sh index b79221885db0..7bb75f718c00 100644 --- a/docker/install/ubuntu2004_install_python_package.sh +++ b/docker/install/ubuntu2004_install_python_package.sh @@ -25,7 +25,6 @@ pip3 install --upgrade \ "Pygments>=2.4.0" \ cloudpickle \ cython \ - decorator \ mypy \ numpy==1.21.* \ orderedset \ diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index ba02b9ffdc5b..e815c25165a1 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -25,7 +25,6 @@ pip3 install --upgrade \ "Pygments>=2.4.0" \ cloudpickle \ cython \ - decorator \ mypy \ numpy==1.21.* \ orderedset \ diff --git a/docs/install/from_source.rst b/docs/install/from_source.rst index 86d11e9a1b36..cc50f9fec93d 100644 --- a/docs/install/from_source.rst +++ b/docs/install/from_source.rst @@ -209,7 +209,7 @@ The following commands can be used to install the extra Python dependencies: .. code:: bash - pip3 install numpy decorator + pip3 install numpy * If you want to use RPC Tracker diff --git a/python/gen_requirements.py b/python/gen_requirements.py index d22544db484b..cfa97ce7479e 100644 --- a/python/gen_requirements.py +++ b/python/gen_requirements.py @@ -64,7 +64,6 @@ "Base requirements needed to install tvm", [ "cloudpickle", - "decorator", "ml_dtypes", "numpy", "packaging", @@ -241,7 +240,6 @@ ("commonmark", ">=0.7.3"), # From PR #213. ("coremltools", None), ("cpplint", None), - ("decorator", None), ( "docutils", "<0.17", diff --git a/python/tvm/_ffi/base.py b/python/tvm/_ffi/base.py index b0a63700b79b..c78d86086e95 100644 --- a/python/tvm/_ffi/base.py +++ b/python/tvm/_ffi/base.py @@ -132,9 +132,9 @@ def decorate(func, fwrapped): fwrapped : function The wrapped function """ - import decorator + from .decorator import decorate as _decorate - return decorator.decorate(func, fwrapped) + return _decorate(func, fwrapped) # ----------------------------------------- diff --git a/python/tvm/_ffi/decorator.py b/python/tvm/_ffi/decorator.py new file mode 100644 index 000000000000..11f0b6531d4f --- /dev/null +++ b/python/tvm/_ffi/decorator.py @@ -0,0 +1,467 @@ +# Copyright (c) 2005-2021, Michele Simionato +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: + +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# Redistributions in bytecode form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. + +# coding: utf-8 +# pylint: disable=all +""" +Decorator module, see +https://github.com/micheles/decorator/blob/master/docs/documentation.md +for the documentation. +""" +import re +import sys +import inspect +import operator +import itertools +from contextlib import _GeneratorContextManager +from inspect import getfullargspec, iscoroutinefunction, isgeneratorfunction + +__version__ = "5.1.1" + +DEF = re.compile(r"\s*def\s*([_\w][_\w\d]*)\s*\(") +POS = inspect.Parameter.POSITIONAL_OR_KEYWORD +EMPTY = inspect.Parameter.empty + + +# this is not used anymore in the core, but kept for backward compatibility +class FunctionMaker(object): + """ + An object with the ability to create functions with a given signature. + It has attributes name, doc, module, signature, defaults, dict and + methods update and make. + """ + + # Atomic get-and-increment provided by the GIL + _compile_count = itertools.count() + + # make pylint happy + args = varargs = varkw = defaults = kwonlyargs = kwonlydefaults = () + + def __init__( + self, + func=None, + name=None, + signature=None, + defaults=None, + doc=None, + module=None, + funcdict=None, + ): + self.shortsignature = signature + if func: + # func can be a class or a callable, but not an instance method + self.name = func.__name__ + if self.name == "": # small hack for lambda functions + self.name = "_lambda_" + self.doc = func.__doc__ + self.module = func.__module__ + if inspect.isroutine(func): + argspec = getfullargspec(func) + self.annotations = getattr(func, "__annotations__", {}) + for a in ("args", "varargs", "varkw", "defaults", "kwonlyargs", "kwonlydefaults"): + setattr(self, a, getattr(argspec, a)) + for i, arg in enumerate(self.args): + setattr(self, "arg%d" % i, arg) + allargs = list(self.args) + allshortargs = list(self.args) + if self.varargs: + allargs.append("*" + self.varargs) + allshortargs.append("*" + self.varargs) + elif self.kwonlyargs: + allargs.append("*") # single star syntax + for a in self.kwonlyargs: + allargs.append("%s=None" % a) + allshortargs.append("%s=%s" % (a, a)) + if self.varkw: + allargs.append("**" + self.varkw) + allshortargs.append("**" + self.varkw) + self.signature = ", ".join(allargs) + self.shortsignature = ", ".join(allshortargs) + self.dict = func.__dict__.copy() + # func=None happens when decorating a caller + if name: + self.name = name + if signature is not None: + self.signature = signature + if defaults: + self.defaults = defaults + if doc: + self.doc = doc + if module: + self.module = module + if funcdict: + self.dict = funcdict + # check existence required attributes + assert hasattr(self, "name") + if not hasattr(self, "signature"): + raise TypeError("You are decorating a non function: %s" % func) + + def update(self, func, **kw): + """ + Update the signature of func with the data in self + """ + func.__name__ = self.name + func.__doc__ = getattr(self, "doc", None) + func.__dict__ = getattr(self, "dict", {}) + func.__defaults__ = self.defaults + func.__kwdefaults__ = self.kwonlydefaults or None + func.__annotations__ = getattr(self, "annotations", None) + try: + frame = sys._getframe(3) + except AttributeError: # for IronPython and similar implementations + callermodule = "?" + else: + callermodule = frame.f_globals.get("__name__", "?") + func.__module__ = getattr(self, "module", callermodule) + func.__dict__.update(kw) + + def make(self, src_templ, evaldict=None, addsource=False, **attrs): + """ + Make a new function from a given template and update the signature + """ + src = src_templ % vars(self) # expand name and signature + evaldict = evaldict or {} + mo = DEF.search(src) + if mo is None: + raise SyntaxError("not a valid function template\n%s" % src) + name = mo.group(1) # extract the function name + names = set([name] + [arg.strip(" *") for arg in self.shortsignature.split(",")]) + for n in names: + if n in ("_func_", "_call_"): + raise NameError("%s is overridden in\n%s" % (n, src)) + + if not src.endswith("\n"): # add a newline for old Pythons + src += "\n" + + # Ensure each generated function has a unique filename for profilers + # (such as cProfile) that depend on the tuple of (, + # , ) being unique. + filename = "" % next(self._compile_count) + try: + code = compile(src, filename, "single") + exec(code, evaldict) + except Exception: + print("Error in generated code:", file=sys.stderr) + print(src, file=sys.stderr) + raise + func = evaldict[name] + if addsource: + attrs["__source__"] = src + self.update(func, **attrs) + return func + + @classmethod + def create( + cls, obj, body, evaldict, defaults=None, doc=None, module=None, addsource=True, **attrs + ): + """ + Create a function from the strings name, signature and body. + evaldict is the evaluation dictionary. If addsource is true an + attribute __source__ is added to the result. The attributes attrs + are added, if any. + """ + if isinstance(obj, str): # "name(signature)" + name, rest = obj.strip().split("(", 1) + signature = rest[:-1] # strip a right parens + func = None + else: # a function + name = None + signature = None + func = obj + self = cls(func, name, signature, defaults, doc, module) + ibody = "\n".join(" " + line for line in body.splitlines()) + caller = evaldict.get("_call_") # when called from `decorate` + if caller and iscoroutinefunction(caller): + body = ("async def %(name)s(%(signature)s):\n" + ibody).replace( + "return", "return await" + ) + else: + body = "def %(name)s(%(signature)s):\n" + ibody + return self.make(body, evaldict, addsource, **attrs) + + +def fix(args, kwargs, sig): + """ + Fix args and kwargs to be consistent with the signature + """ + ba = sig.bind(*args, **kwargs) + ba.apply_defaults() # needed for test_dan_schult + return ba.args, ba.kwargs + + +def decorate(func, caller, extras=(), kwsyntax=False): + """ + Decorates a function/generator/coroutine using a caller. + If kwsyntax is True calling the decorated functions with keyword + syntax will pass the named arguments inside the ``kw`` dictionary, + even if such argument are positional, similarly to what functools.wraps + does. By default kwsyntax is False and the the arguments are untouched. + """ + sig = inspect.signature(func) + if iscoroutinefunction(caller): + + async def fun(*args, **kw): + if not kwsyntax: + args, kw = fix(args, kw, sig) + return await caller(func, *(extras + args), **kw) + + elif isgeneratorfunction(caller): + + def fun(*args, **kw): + if not kwsyntax: + args, kw = fix(args, kw, sig) + for res in caller(func, *(extras + args), **kw): + yield res + + else: + + def fun(*args, **kw): + if not kwsyntax: + args, kw = fix(args, kw, sig) + return caller(func, *(extras + args), **kw) + + fun.__name__ = func.__name__ + fun.__doc__ = func.__doc__ + fun.__wrapped__ = func + fun.__signature__ = sig + fun.__qualname__ = func.__qualname__ + # builtin functions like defaultdict.__setitem__ lack many attributes + try: + fun.__defaults__ = func.__defaults__ + except AttributeError: + pass + try: + fun.__kwdefaults__ = func.__kwdefaults__ + except AttributeError: + pass + try: + fun.__annotations__ = func.__annotations__ + except AttributeError: + pass + try: + fun.__module__ = func.__module__ + except AttributeError: + pass + try: + fun.__dict__.update(func.__dict__) + except AttributeError: + pass + return fun + + +def decoratorx(caller): + """ + A version of "decorator" implemented via "exec" and not via the + Signature object. Use this if you are want to preserve the `.__code__` + object properties (https://github.com/micheles/decorator/issues/129). + """ + + def dec(func): + return FunctionMaker.create( + func, + "return _call_(_func_, %(shortsignature)s)", + dict(_call_=caller, _func_=func), + __wrapped__=func, + __qualname__=func.__qualname__, + ) + + return dec + + +def decorator(caller, _func=None, kwsyntax=False): + """ + decorator(caller) converts a caller function into a decorator + """ + if _func is not None: # return a decorated function + # this is obsolete behavior; you should use decorate instead + return decorate(_func, caller, (), kwsyntax) + # else return a decorator function + sig = inspect.signature(caller) + dec_params = [p for p in sig.parameters.values() if p.kind is POS] + + def dec(func=None, *args, **kw): + na = len(args) + 1 + extras = args + tuple( + kw.get(p.name, p.default) for p in dec_params[na:] if p.default is not EMPTY + ) + if func is None: + return lambda func: decorate(func, caller, extras, kwsyntax) + else: + return decorate(func, caller, extras, kwsyntax) + + dec.__signature__ = sig.replace(parameters=dec_params) + dec.__name__ = caller.__name__ + dec.__doc__ = caller.__doc__ + dec.__wrapped__ = caller + dec.__qualname__ = caller.__qualname__ + dec.__kwdefaults__ = getattr(caller, "__kwdefaults__", None) + dec.__dict__.update(caller.__dict__) + return dec + + +# ####################### contextmanager ####################### # + + +class ContextManager(_GeneratorContextManager): + def __init__(self, g, *a, **k): + _GeneratorContextManager.__init__(self, g, a, k) + + def __call__(self, func): + def caller(f, *a, **k): + with self.__class__(self.func, *self.args, **self.kwds): + return f(*a, **k) + + return decorate(func, caller) + + +_contextmanager = decorator(ContextManager) + + +def contextmanager(func): + # Enable Pylint config: contextmanager-decorators=decorator.contextmanager + return _contextmanager(func) + + +# ############################ dispatch_on ############################ # + + +def append(a, vancestors): + """ + Append ``a`` to the list of the virtual ancestors, unless it is already + included. + """ + add = True + for j, va in enumerate(vancestors): + if issubclass(va, a): + add = False + break + if issubclass(a, va): + vancestors[j] = a + add = False + if add: + vancestors.append(a) + + +# inspired from simplegeneric by P.J. Eby and functools.singledispatch +def dispatch_on(*dispatch_args): + """ + Factory of decorators turning a function into a generic function + dispatching on the given arguments. + """ + assert dispatch_args, "No dispatch args passed" + dispatch_str = "(%s,)" % ", ".join(dispatch_args) + + def check(arguments, wrong=operator.ne, msg=""): + """Make sure one passes the expected number of arguments""" + if wrong(len(arguments), len(dispatch_args)): + raise TypeError( + "Expected %d arguments, got %d%s" % (len(dispatch_args), len(arguments), msg) + ) + + def gen_func_dec(func): + """Decorator turning a function into a generic function""" + + # first check the dispatch arguments + argset = set(getfullargspec(func).args) + if not set(dispatch_args) <= argset: + raise NameError("Unknown dispatch arguments %s" % dispatch_str) + + typemap = {} + + def vancestors(*types): + """ + Get a list of sets of virtual ancestors for the given types + """ + check(types) + ras = [[] for _ in range(len(dispatch_args))] + for types_ in typemap: + for t, type_, ra in zip(types, types_, ras): + if issubclass(t, type_) and type_ not in t.mro(): + append(type_, ra) + return [set(ra) for ra in ras] + + def ancestors(*types): + """ + Get a list of virtual MROs, one for each type + """ + check(types) + lists = [] + for t, vas in zip(types, vancestors(*types)): + n_vas = len(vas) + if n_vas > 1: + raise RuntimeError("Ambiguous dispatch for %s: %s" % (t, vas)) + elif n_vas == 1: + (va,) = vas + mro = type("t", (t, va), {}).mro()[1:] + else: + mro = t.mro() + lists.append(mro[:-1]) # discard t and object + return lists + + def register(*types): + """ + Decorator to register an implementation for the given types + """ + check(types) + + def dec(f): + check(getfullargspec(f).args, operator.lt, " in " + f.__name__) + typemap[types] = f + return f + + return dec + + def dispatch_info(*types): + """ + An utility to introspect the dispatch algorithm + """ + check(types) + lst = [] + for anc in itertools.product(*ancestors(*types)): + lst.append(tuple(a.__name__ for a in anc)) + return lst + + def _dispatch(dispatch_args, *args, **kw): + types = tuple(type(arg) for arg in dispatch_args) + try: # fast path + f = typemap[types] + except KeyError: + pass + else: + return f(*args, **kw) + combinations = itertools.product(*ancestors(*types)) + next(combinations) # the first one has been already tried + for types_ in combinations: + f = typemap.get(types_) + if f is not None: + return f(*args, **kw) + + # else call the default implementation + return func(*args, **kw) + + return FunctionMaker.create( + func, + "return _f_(%s, %%(shortsignature)s)" % dispatch_str, + dict(_f_=_dispatch), + register=register, + default=func, + typemap=typemap, + vancestors=vancestors, + ancestors=ancestors, + dispatch_info=dispatch_info, + __wrapped__=func, + ) + + gen_func_dec.__name__ = "dispatch_on" + dispatch_str + return gen_func_dec diff --git a/tests/lint/rat-excludes b/tests/lint/rat-excludes index 08250b9fb069..feeaeb9ecb4d 100644 --- a/tests/lint/rat-excludes +++ b/tests/lint/rat-excludes @@ -52,3 +52,6 @@ poetry.lock # Template files for Jenkins .*\.groovy\.j2 + +# 3rdparty packages +decorator.py