Metadata validation API

MANIFEST.in

@@ -8,6 +8,7 @@ include .pre-commit-config.yaml
 recursive-include docs *
 recursive-include tests *.py
 recursive-include tests hello-world-*
+recursive-include packaging *.py
 
 exclude noxfile.py
 exclude .travis.yml

noxfile.py

@@ -26,7 +26,8 @@ def coverage(*args):
         session.run("python", "-m", "coverage", *args)
 
     session.install("coverage<5.0.0", "pretend", "pytest", "pip>=9.0.2")
-
+    session.install("typing_extensions")
+
     if "pypy" not in session.python:
         coverage(
             "run",

packaging/metadata/__init__.py

@@ -0,0 +1,136 @@
+from email.parser import HeaderParser
+from email.message import Message
+from typing import Dict, Iterator, Union, List, Any
+from typing_extensions import TypedDict
+import inspect
+import json
+from .constants import VERSIONED_METADATA_FIELDS
+import sys
+
+
+def _json_form(val: str) -> str:
+    return val.lower().replace("-", "_")
+
+
+def _canonicalize(
+    metadata: Dict[str, Union[List[str], str]]
+) -> Dict[str, Union[List[str], str]]:
+    """
+    Transforms a metadata object to the canonical representation
+    as specified in
+    https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
+    All transformed keys should be reduced to lower case. Hyphens
+    should be replaced with underscores, but otherwise should retain all
+    other characters.
+    """
+    return {_json_form(key): value for key, value in metadata.items()}
+
+
+def check_python_compatability() -> None:
+    if sys.version_info[0] < 3:
+        raise ModuleNotFoundError()
+
+
+check_python_compatability()
+
+
+class Metadata:
+    def __init__(self, **kwargs: Union[List[str], str]) -> None:
+        self._meta_dict = kwargs
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Metadata):
+            return self._meta_dict == other._meta_dict
+        return NotImplemented
+
+    @classmethod
+    def from_json(cls, data: str) -> "Metadata":
+        return cls(**_canonicalize(json.loads(data)))
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Union[List[str], str]]) -> "Metadata":
+        return cls(**_canonicalize(data))
+
+    @classmethod
+    def from_rfc822(cls, rfc822_string: str) -> "Metadata":
+        return cls(**Metadata._rfc822_string_to_dict(rfc822_string))
+
+    def to_json(self) -> str:
+        return json.dumps(self._meta_dict, sort_keys=True)
+
+    def to_dict(self) -> Dict:
+        return self._meta_dict
+
+    def to_rfc822(self) -> str:
+        msg = Message()
+        metadata_version = self._meta_dict["metadata_version"]
+        metadata_fields = VERSIONED_METADATA_FIELDS[metadata_version]
+        for field in (
+            metadata_fields["SINGLE"]
+            | metadata_fields["MULTI"]
+            | metadata_fields["TREAT_AS_MULTI"]
+        ):
+            value = self._meta_dict.get(_json_form(field))
+            if value:
+                if field == "Description":
+                    # Special case - put in payload
+                    msg.set_payload(value)
+                    continue
+                if field == "Keywords":
+                    value = ",".join(value)
+                if isinstance(value, str):
+                    value = [value]
+                for item in value:
+                    msg.add_header(field, item)
+
+        return msg.as_string()
+
+    def __iter__(self) -> Iterator[Any]:
+        return iter(self._meta_dict.items())
+
+    @classmethod
+    def _rfc822_string_to_dict(
+        cls, rfc822_string: str
+    ) -> Dict[str, Union[List[str], str]]:
+        """Extracts metadata information from a metadata-version 2.1 object.
+
+        https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
+
+        - The original key-value format should be read with email.parser.HeaderParser;
+        - All transformed keys should be reduced to lower case. Hyphens should
+          be replaced with underscores, but otherwise should retain all other
+          characters;
+        - The transformed value for any field marked with "(Multiple-use")
+          should be a single list containing all the original values for the
+          given key;
+        - The Keywords field should be converted to a list by splitting the
+          original value on whitespace characters;
+        - The message body, if present, should be set to the value of the
+          description key.
+        - The result should be stored as a string-keyed dictionary.
+        """
+        metadata: Dict[str, Union[List[str], str]] = {}
+        parsed = HeaderParser().parsestr(rfc822_string)
+        metadata_fields = VERSIONED_METADATA_FIELDS[parsed.get("Metadata-Version")]
+
+        for key, value in parsed.items():
+            if key in metadata_fields["MULTI"]:
+                metadata.setdefault(key, []).append(value)
+            elif key in metadata_fields["TREAT_AS_MULTI"]:
+                metadata[key] = [val.strip() for val in value.split(",")]
+            elif key == "Description":
+                metadata[key] = inspect.cleandoc(value)
+            else:
+                metadata[key] = value
+
+        # Handle the message payload
+        payload = parsed.get_payload()
+        if payload:
+            if "Description" in metadata:
+                print("Both Description and payload given - ignoring Description")
+            metadata["Description"] = payload
+
+        return _canonicalize(metadata)
+
+    def validate(self) -> bool:
+        raise NotImplementedError

packaging/metadata/constants.py

@@ -0,0 +1,121 @@
+MULTI_1_0 = {"Platform"}  # type :  typing.Set[str]
+
+TREAT_AS_MULTI_1_0 = {"Keywords"}  # type : typing.Set[str]
+
+SINGLE_1_0 = {
+    "Metadata-Version",
+    "Name",
+    "Version",
+    "Summary",
+    "Description",
+    "Home-page",
+    "Author",
+    "Author-email",
+    "License",
+}  # type : typing.Set[str]
+
+
+MULTI_1_1 = {"Platform", "Supported-Platform", "Classifier"}  # type : typing.Set[str]
+
+TREAT_AS_MULTI_1_1 = {"Keywords"}  # type : typing.Set[str]
+
+SINGLE_1_1 = {
+    "Metadata-Version",
+    "Name",
+    "Version",
+    "Summary",
+    "Description",
+    "Home-page",
+    "Download-URL",
+    "Author",
+    "Author-email",
+    "License",
+}  # type : typing.Set[str]
+
+
+MULTI_1_2 = {
+    "Platform",
+    "Supported-Platform",
+    "Classifier",
+    "Requires-Dist",
+    "Provides-Dist",
+    "Obsoletes-Dist",
+    "Requires-External",
+    "Project-URL",
+}  # type : typing.Set[str]
+
+TREAT_AS_MULTI_1_2 = {"Keywords"}  # type : typing.Set[str]
+
+SINGLE_1_2 = {
+    "Metadata-Version",
+    "Name",
+    "Version",
+    "Summary",
+    "Description",
+    "Home-page",
+    "Download-URL",
+    "Author",
+    "Author-email",
+    "Maintainer",
+    "Maintainer-email",
+    "License",
+    "Requires-Python",
+}  # type : typing.Set[str]
+
+
+MULTI_2_1 = {
+    "Platform",
+    "Supported-Platform",
+    "Classifier",
+    "Requires-Dist",
+    "Provides-Dist",
+    "Obsoletes-Dist",
+    "Requires-External",
+    "Project-URL",
+    "Provides-Extra",
+}  # type : typing.Set[str]
+
+TREAT_AS_MULTI_2_1 = {"Keywords"}  # type : typing.Set[str]
+
+SINGLE_2_1 = {
+    "Metadata-Version",
+    "Name",
+    "Version",
+    "Summary",
+    "Description",
+    "Description-Content-Type",
+    "Home-page",
+    "Download-URL",
+    "Author",
+    "Author-email",
+    "Maintainer",
+    "Maintainer-email",
+    "License",
+    "Requires-Python",
+}  # type : typing.Set[str]
+
+
+VERSIONED_METADATA_FIELDS = {
+    "1.0": {
+        "MULTI": MULTI_1_0,
+        "TREAT_AS_MULTI": TREAT_AS_MULTI_1_0,
+        "SINGLE": SINGLE_1_0,
+    },
+    "1.1": {
+        "MULTI": MULTI_1_1,
+        "TREAT_AS_MULTI": TREAT_AS_MULTI_1_1,
+        "SINGLE": SINGLE_1_1,
+    },
+    "1.2": {
+        "MULTI": MULTI_1_2,
+        "TREAT_AS_MULTI": TREAT_AS_MULTI_1_2,
+        "SINGLE": SINGLE_1_2,
+    },
+    "2.1": {
+        "MULTI": MULTI_2_1,
+        "TREAT_AS_MULTI": TREAT_AS_MULTI_2_1,
+        "SINGLE": SINGLE_2_1,
+    },
+}  # type : typing.Any
+
+# typing.Dict[typing.Union[typing.List[str],str], typing.Dict[str, typing.Set[str]]]

tests/metadata/2_1_pkginfo_string.txt

@@ -0,0 +1,71 @@
+Metadata-Version: 2.1
+Name: sampleproject
+Version: 2.0.0
+Summary: A sample Python project
+Home-page: https://github.com/pypa/sampleproject
+Author: A. Random Developer
+Author-email: author@example.com
+License: UNKNOWN
+Project-URL: Bug Reports, https://github.com/pypa/sampleproject/issues
+Project-URL: Funding, https://donate.pypi.org
+Project-URL: Say Thanks!, http://saythanks.io/to/example
+Project-URL: Source, https://github.com/pypa/sampleproject/
+Description: # A sample Python project
+
+        ![Python Logo](https://www.python.org/static/community_logos/python-logo.png "Sample inline image")
+
+        A sample project that exists as an aid to the [Python Packaging User
+        Guide][packaging guide]'s [Tutorial on Packaging and Distributing
+        Projects][distribution tutorial].
+
+        This project does not aim to cover best practices for Python project
+        development as a whole. For example, it does not provide guidance or tool
+        recommendations for version control, documentation, or testing.
+
+        [The source for this project is available here][src].
+
+        Most of the configuration for a Python project is done in the `setup.py` file,
+        an example of which is included in this project. You should edit this file
+        accordingly to adapt this sample project to your needs.
+
+        ----
+
+        This is the README file for the project.
+
+        The file should use UTF-8 encoding and can be written using
+        [reStructuredText][rst] or [markdown][md use] with the appropriate [key set][md
+        use]. It will be used to generate the project webpage on PyPI and will be
+        displayed as the project homepage on common code-hosting services, and should be
+        written for that purpose.
+
+        Typical contents for this file would include an overview of the project, basic
+        usage examples, etc. Generally, including the project changelog in here is not a
+        good idea, although a simple “What's New” section for the most recent version
+        may be appropriate.
+
+        [packaging guide]: https://packaging.python.org
+        [distribution tutorial]: https://packaging.python.org/tutorials/packaging-projects/
+        [src]: https://github.com/pypa/sampleproject
+        [rst]: http://docutils.sourceforge.net/rst.html
+        [md]: https://tools.ietf.org/html/rfc7764#section-3.5 "CommonMark variant"
+        [md use]: https://packaging.python.org/specifications/core-metadata/#description-content-type-optional
+
+Keywords: sample,setuptools,development
+Platform: UNKNOWN
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.5, <4
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Provides-Extra: test
+Requires-Dist: peppercorn
+Requires-Dist: check-manifest ; extra == 'dev'
+Requires-Dist: coverage ; extra == 'test'