From e2722e96cc976489b61259a6872de7762cbb20f3 Mon Sep 17 00:00:00 2001 From: Nathan J Mehl Date: Sun, 24 Jul 2022 15:04:40 -0400 Subject: [PATCH] Add support for zstandard control archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add zstandard to poetry.lock - add test coverage - bump version to 1.7.0 Co-authored-by: Yunusemre Şentürk --- Makefile | 2 +- poetry.lock | 66 ++++++++++++++++++++++++++-- pydpkg/dpkg.py | 15 ++++++- pydpkg/exceptions.py | 4 +- pyproject.toml | 3 +- tests/sample_package_badcontrol.deb | Bin 0 -> 694 bytes tests/sample_package_zst.deb | Bin 0 -> 674 bytes tests/test_dpkg.py | 48 ++++++++++++++++++-- 8 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 tests/sample_package_badcontrol.deb create mode 100644 tests/sample_package_zst.deb diff --git a/Makefile b/Makefile index 154fa9d..d82bf30 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ else endif ${PYENV}: ${BREW_SSL} ${BREW_READLINE} ${PYENV_BIN} - ${ARCH_PREFIX} pyenv install ${PYVERSION} + ${ARCH_PREFIX} pyenv install -s ${PYVERSION} ${VENV}: ${PYENV} ${ARCH_PREFIX} ${PYENV_BIN} virtualenv ${PYVERSION} ${VENV_NAME} diff --git a/poetry.lock b/poetry.lock index 02949ab..6c5cd87 100644 --- a/poetry.lock +++ b/poetry.lock @@ -281,8 +281,8 @@ optional = false python-versions = ">=3.6" [package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] +test = ["pytest-mock (>=3.6)", "pytest-cov (>=2.7)", "pytest (>=6)", "appdirs (==1.4.4)"] +docs = ["sphinx-autodoc-typehints (>=1.12)", "proselint (>=0.10.2)", "furo (>=2021.7.5b38)", "Sphinx (>=4)"] [[package]] name = "pluggy" @@ -478,10 +478,24 @@ python-versions = ">=3.6" docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +[[package]] +name = "zstandard" +version = "0.18.0" +description = "Zstandard bindings for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = {version = ">=1.11", markers = "platform_python_implementation == \"PyPy\""} + +[package.extras] +cffi = ["cffi (>=1.11)"] + [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "e2b8e67c9c15067ad778f0db14c9e1499b8cab658ca64f49e0329a1bb90bffc7" +content-hash = "d7a19d8e51dc34487e8016e012e93d4ec3e0b82008923e4810580cb4b3dd6923" [metadata.files] arpy = [ @@ -884,3 +898,49 @@ zipp = [ {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, ] +zstandard = [ + {file = "zstandard-0.18.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"}, + {file = "zstandard-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b"}, + {file = "zstandard-0.18.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32"}, + {file = "zstandard-0.18.0-cp310-cp310-win32.whl", hash = "sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5"}, + {file = "zstandard-0.18.0-cp310-cp310-win_amd64.whl", hash = "sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435"}, + {file = "zstandard-0.18.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244"}, + {file = "zstandard-0.18.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63"}, + {file = "zstandard-0.18.0-cp36-cp36m-win32.whl", hash = "sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67"}, + {file = "zstandard-0.18.0-cp36-cp36m-win_amd64.whl", hash = "sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d"}, + {file = "zstandard-0.18.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722"}, + {file = "zstandard-0.18.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0"}, + {file = "zstandard-0.18.0-cp37-cp37m-win32.whl", hash = "sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936"}, + {file = "zstandard-0.18.0-cp37-cp37m-win_amd64.whl", hash = "sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c"}, + {file = "zstandard-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb"}, + {file = "zstandard-0.18.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e"}, + {file = "zstandard-0.18.0-cp38-cp38-win32.whl", hash = "sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab"}, + {file = "zstandard-0.18.0-cp38-cp38-win_amd64.whl", hash = "sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a"}, + {file = "zstandard-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0"}, + {file = "zstandard-0.18.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25"}, + {file = "zstandard-0.18.0-cp39-cp39-win32.whl", hash = "sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784"}, + {file = "zstandard-0.18.0-cp39-cp39-win_amd64.whl", hash = "sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c"}, + {file = "zstandard-0.18.0.tar.gz", hash = "sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f"}, +] diff --git a/pydpkg/dpkg.py b/pydpkg/dpkg.py index 2ec48ed..1475919 100644 --- a/pydpkg/dpkg.py +++ b/pydpkg/dpkg.py @@ -13,6 +13,7 @@ # pypi imports import six +import zstandard from arpy import Archive # local imports @@ -255,9 +256,12 @@ def _process_dpkg_file(self, filename): elif b"control.tar.xz" in dpkg_archive.archived_files: control_archive = dpkg_archive.archived_files[b"control.tar.xz"] control_archive_type = "xz" + elif b"control.tar.zst" in dpkg_archive.archived_files: + control_archive = dpkg_archive.archived_files[b"control.tar.zst"] + control_archive_type = "zst" else: raise DpkgMissingControlGzipFile( - "Corrupt dpkg file: no control.tar.gz/xz file in ar archive." + "Corrupt dpkg file: no control.tar.gz/xz/zst file in ar archive." ) self._log.debug("found controlgz: %s", control_archive) @@ -267,12 +271,19 @@ def _process_dpkg_file(self, filename): with tarfile.open(fileobj=io.BytesIO(gzf.read())) as ctar: self._log.debug("opened tar file: %s", ctar) message = self._extract_message(ctar) - else: + elif control_archive_type == "xz": with lzma.open(control_archive) as xzf: self._log.debug("opened xz control archive: %s", xzf) with tarfile.open(fileobj=io.BytesIO(xzf.read())) as ctar: self._log.debug("opened tar file: %s", ctar) message = self._extract_message(ctar) + else: + zst = zstandard.ZstdDecompressor() + with zst.stream_reader(control_archive) as reader: + self._log.debug("opened zst control archive: %s", reader) + with tarfile.open(fileobj=io.BytesIO(reader.read())) as ctar: + self._log.debug("opened tar file: %s", ctar) + message = self._extract_message(ctar) for req in REQUIRED_HEADERS: if req not in list(map(str.lower, message.keys())): diff --git a/pydpkg/exceptions.py b/pydpkg/exceptions.py index b60fdb8..8172292 100644 --- a/pydpkg/exceptions.py +++ b/pydpkg/exceptions.py @@ -14,11 +14,11 @@ class DpkgVersionError(DpkgError): class DpkgMissingControlFile(DpkgError): - """No control file found in control.tar.gz/xz""" + """No control file found in control.tar.gz/xz/zst""" class DpkgMissingControlGzipFile(DpkgError): - """No control.tar.gz/xz file found in dpkg file""" + """No control.tar.gz/xz/zst file found in dpkg file""" class DpkgMissingRequiredHeaderError(DpkgError): diff --git a/pyproject.toml b/pyproject.toml index 15551ce..6b4a96d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pydpkg" -version = "1.6.0" +version = "1.7.0" description = "A python library for parsing debian package control headers and version strings" authors = ["Nathan J. Mehl "] homepage = "https://github.com/memory/python-dpkg" @@ -31,6 +31,7 @@ python = "^3.6.2" arpy = "^2.2.0" six = "^1.16.0" PGPy = "0.5.4" +zstandard = "^0.18.0" [tool.poetry.dev-dependencies] black = "^21.11b1" diff --git a/tests/sample_package_badcontrol.deb b/tests/sample_package_badcontrol.deb new file mode 100644 index 0000000000000000000000000000000000000000..86bd64d44bb0702e9d7c8245670c1006db5f97c1 GIT binary patch literal 694 zcmY$iNi0gvu;WTeP0CEn(@o0EODw8XP*5;5Gqo_Y00J`w1ych#dL9VV% zVVYYNZv5MLSHbcATaT-^&$C5!aj>-AbUnR^D{FcrE9={LQD@%iNvSY$EJ@_JeO_XM z-6mz}P0hTgWR!(+Hr_uS7WU!%@1Kf)pZKddzudC2N3{3htwhrtvx%zGlP2YCx;#_C z^4^4-GS!OnjlLbT{uE6#jdKcW#;M1& literal 0 HcmV?d00001 diff --git a/tests/sample_package_zst.deb b/tests/sample_package_zst.deb new file mode 100644 index 0000000000000000000000000000000000000000..e143f9a9293779f9fd9a800da7ddb23dee7affa4 GIT binary patch literal 674 zcmY$iNi0gvu;WTeP0CEn(@o0EODw8XP*5;5Gqo_YurRSSQBW{7Fa(Mi82||dLjwad z6QCFnK|unSk)8opa(-S(QGSkINn(*+RdESH{l*3`6B4*Iw(9>)VNe(4WSAyst`N|E zjnScXO8T`&-1gg?9=7)$5K%cTE%~u2SI+pi+*Kw$23;WZ5ajCG6sEaV;l{s>cNHA( zzxB9!`#f7z7Y9q*P1n<#xU!~4va-H?7j@>Ho|FnB$C5;j+vg=F*lkjl-qg%{N=8{I zXXE|TVPPN6|Ng1?_lduX^UEzOdqjI5-byshF`K9=J!w+Trpq%GEbmRYDO0UD-{{*h z>wnt|_$JJmYI*#+-9EJmSJt@(9y`zdvXV#O)QwE$mz6I{R@C%6iSpc3*eD*a5c=En zV6Rs7J2h#c`p@$_kEQ$&dYd81FqP>8!wSL9HZG6ZJoB<-So$^}nHo|3$VtvaQ9<|M zL`xlm1w{(|jGLVIF%={@Ni9gwkYRYp?_d_ecwl3ZGQW8rN;Q