diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abd6b88fee..115e8a6482 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,6 +79,11 @@ Changed Contributed by @LiamRiddell +* Use PEP 440 direct reference requirements instead of legacy PIP VCS requirements. Now, our ``*.requirements.txt`` files use + ``package-name@ git+https://url@version ; markers`` instead of ``git+https://url@version#egg=package-name ; markers``. #5673 + + Contributed by @cognifloyd + Removed ~~~~~~~ diff --git a/contrib/linux/requirements.txt b/contrib/linux/requirements.txt index 597fda989a..d1864ccbb8 100644 --- a/contrib/linux/requirements.txt +++ b/contrib/linux/requirements.txt @@ -1,3 +1,3 @@ # used by file watcher sensor -pyinotify>=0.9.5,<=0.10 --e git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper +pyinotify>=0.9.5,<=0.10 ; platform_system=="Linux" +logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" diff --git a/contrib/runners/action_chain_runner/dist_utils.py b/contrib/runners/action_chain_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/action_chain_runner/dist_utils.py +++ b/contrib/runners/action_chain_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/announcement_runner/dist_utils.py b/contrib/runners/announcement_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/announcement_runner/dist_utils.py +++ b/contrib/runners/announcement_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/http_runner/dist_utils.py b/contrib/runners/http_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/http_runner/dist_utils.py +++ b/contrib/runners/http_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/inquirer_runner/dist_utils.py b/contrib/runners/inquirer_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/inquirer_runner/dist_utils.py +++ b/contrib/runners/inquirer_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/local_runner/dist_utils.py b/contrib/runners/local_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/local_runner/dist_utils.py +++ b/contrib/runners/local_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/noop_runner/dist_utils.py b/contrib/runners/noop_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/noop_runner/dist_utils.py +++ b/contrib/runners/noop_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/orquesta_runner/dist_utils.py b/contrib/runners/orquesta_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/orquesta_runner/dist_utils.py +++ b/contrib/runners/orquesta_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/orquesta_runner/in-requirements.txt b/contrib/runners/orquesta_runner/in-requirements.txt index f7583eb2bc..3302e48fad 100644 --- a/contrib/runners/orquesta_runner/in-requirements.txt +++ b/contrib/runners/orquesta_runner/in-requirements.txt @@ -1 +1 @@ -git+https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta +orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 diff --git a/contrib/runners/orquesta_runner/requirements.txt b/contrib/runners/orquesta_runner/requirements.txt index b8d9afb3f8..be64688128 100644 --- a/contrib/runners/orquesta_runner/requirements.txt +++ b/contrib/runners/orquesta_runner/requirements.txt @@ -5,4 +5,4 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -git+https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta +orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 diff --git a/contrib/runners/python_runner/dist_utils.py b/contrib/runners/python_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/python_runner/dist_utils.py +++ b/contrib/runners/python_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/remote_runner/dist_utils.py b/contrib/runners/remote_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/remote_runner/dist_utils.py +++ b/contrib/runners/remote_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/contrib/runners/winrm_runner/dist_utils.py b/contrib/runners/winrm_runner/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/contrib/runners/winrm_runner/dist_utils.py +++ b/contrib/runners/winrm_runner/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/fixed-requirements.txt b/fixed-requirements.txt index eb3b9adf3e..ef087fd789 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -39,7 +39,7 @@ oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 passlib==1.7.4 prompt-toolkit==1.0.15 -pyinotify==0.9.6; platform_system=="Linux" +pyinotify==0.9.6 ; platform_system=="Linux" pymongo==3.11.3 pyparsing<3 zstandard==0.15.2 diff --git a/requirements.txt b/requirements.txt index e3890d0b80..50e7cb7f01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,11 +18,6 @@ decorator==4.4.2 dnspython>=1.16.0,<2.0.0 eventlet==0.30.2 flex==6.14.1 -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper -git+https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta -git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file -git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap -git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitdb==4.0.2 gitpython==3.1.15 greenlet==1.0.0 @@ -33,6 +28,7 @@ jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==5.0.2 lockfile==0.12.2 +logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" mock==4.0.3 mongoengine==0.23.0 networkx>=2.5.1,<2.6 @@ -40,6 +36,7 @@ nose nose-parallel==0.4.0 nose-timer==1.0.1 orjson==3.5.2 +orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 @@ -48,7 +45,7 @@ prettytable==2.1.0 prompt-toolkit==1.0.15 psutil==5.8.0 pyOpenSSL<=21.0.0 -pyinotify==0.9.6; platform_system=="Linux" +pyinotify==0.9.6 ; platform_system=="Linux" pymongo==3.11.3 pyparsing<3 pyrabbit @@ -69,6 +66,9 @@ semver==2.13.0 simplejson six==1.13.0 sseclient-py==1.7 +st2-auth-backend-flat-file@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master +st2-auth-ldap@ git+https://github.com/StackStorm/st2-auth-ldap.git@master +st2-rbac-backend@ git+https://github.com/StackStorm/st2-rbac-backend.git@master stevedore==1.30.1 tenacity>=3.2.1,<7.0.0 tooz==2.8.0 diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index c0af527b6b..297efe1689 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -122,6 +122,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/scripts/fixate-requirements.py b/scripts/fixate-requirements.py index 1bd6e8e91b..a60f43b4aa 100755 --- a/scripts/fixate-requirements.py +++ b/scripts/fixate-requirements.py @@ -225,7 +225,14 @@ def write_requirements( # we don't have any idea how to process links, so just add them if linkreq.link and linkreq.link not in links: links.add(linkreq.link) - rline = str(linkreq.link) + if hasattr(req, "req") and req.req and str(req.req).count("@") == 2: + # PEP 440 direct reference + rline = str(linkreq.req) + # Also write out environment markers + if linkreq.markers: + rline += " ; {}".format(str(linkreq.markers)) + else: + rline = str(linkreq.link) if (hasattr(req, "is_editable") and req.is_editable) or ( hasattr(req, "editable") and req.editable diff --git a/st2actions/dist_utils.py b/st2actions/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2actions/dist_utils.py +++ b/st2actions/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2actions/in-requirements.txt b/st2actions/in-requirements.txt index 20599f5d7b..14cda20b57 100644 --- a/st2actions/in-requirements.txt +++ b/st2actions/in-requirements.txt @@ -18,7 +18,7 @@ gitpython lockfile # needed by core "linux" pack - TODO: create virtualenv for linux pack on postinst pyinotify -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper +logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" # required by pack_mgmt/setup_virtualenv.py#L135 virtualenv # needed by requests diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index e3b796a501..acd17a961e 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -9,14 +9,14 @@ MarkupSafe<2.1.0,>=0.23 apscheduler==3.7.0 chardet<3.1.0 eventlet==0.30.2 -git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper gitpython==3.1.15 jinja2==2.11.3 kombu==5.0.2 lockfile==0.12.2 +logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" oslo.config>=1.12.1,<1.13 oslo.utils<5.0,>=4.0.0 -pyinotify==0.9.6; platform_system=="Linux" +pyinotify==0.9.6 ; platform_system=="Linux" pyparsing<3 python-dateutil==2.8.1 python-json-logger diff --git a/st2api/dist_utils.py b/st2api/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2api/dist_utils.py +++ b/st2api/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2auth/dist_utils.py b/st2auth/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2auth/dist_utils.py +++ b/st2auth/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2auth/in-requirements.txt b/st2auth/in-requirements.txt index d6de70f4f8..0d9e5e01a3 100644 --- a/st2auth/in-requirements.txt +++ b/st2auth/in-requirements.txt @@ -7,6 +7,6 @@ pymongo six stevedore # For backward compatibility reasons, flat file backend is installed by default -git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file -git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap +st2-auth-backend-flat-file@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master +st2-auth-ldap@ git+https://github.com/StackStorm/st2-auth-ldap.git@master gunicorn diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index 6b6016bcb6..1d6a06de81 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -7,11 +7,11 @@ # update the component requirements.txt bcrypt==3.2.0 eventlet==0.30.2 -git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file -git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap gunicorn==20.1.0 oslo.config>=1.12.1,<1.13 passlib==1.7.4 pymongo==3.11.3 six==1.13.0 +st2-auth-backend-flat-file@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master +st2-auth-ldap@ git+https://github.com/StackStorm/st2-auth-ldap.git@master stevedore==1.30.1 diff --git a/st2client/dist_utils.py b/st2client/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2client/dist_utils.py +++ b/st2client/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2common/dist_utils.py b/st2common/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2common/dist_utils.py +++ b/st2common/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index 921ee720f3..7dde4932a1 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -14,8 +14,8 @@ mongoengine networkx # used by networkx decorator -git+https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta -git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend +orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 +st2-rbac-backend@ git+https://github.com/StackStorm/st2-rbac-backend.git@master oslo.config paramiko pyyaml diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 268a88acf7..dc16e87b5f 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -15,8 +15,6 @@ decorator==4.4.2 dnspython>=1.16.0,<2.0.0 eventlet==0.30.2 flex==6.14.1 -git+https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta -git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitdb==4.0.2 gitpython==3.1.15 greenlet==1.0.0 @@ -28,6 +26,7 @@ lockfile==0.12.2 mongoengine==0.23.0 networkx>=2.5.1,<2.6 orjson==3.5.2 +orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 oslo.config>=1.12.1,<1.13 paramiko==2.10.1 pyOpenSSL<=21.0.0 @@ -41,6 +40,7 @@ retrying==1.3.3 routes==2.4.1 semver==2.13.0 six==1.13.0 +st2-rbac-backend@ git+https://github.com/StackStorm/st2-rbac-backend.git@master tenacity>=3.2.1,<7.0.0 tooz==2.8.0 udatetime==0.0.16 diff --git a/st2reactor/dist_utils.py b/st2reactor/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2reactor/dist_utils.py +++ b/st2reactor/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2stream/dist_utils.py b/st2stream/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2stream/dist_utils.py +++ b/st2stream/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None diff --git a/st2tests/dist_utils.py b/st2tests/dist_utils.py index 2f2043cf29..a4e9862d2b 100644 --- a/st2tests/dist_utils.py +++ b/st2tests/dist_utils.py @@ -118,6 +118,12 @@ def _get_link(line): link = line.replace("-e ", "").strip() return link, req_name[0] + elif vcs_prefix in line and line.count("@") == 2: + # PEP 440 direct reference: @ @version + req_name, link = line.split("@", 1) + req_name = req_name.strip() + link = f"{link.strip()}#egg={req_name}" + return link, req_name return None, None