From 1c1e3ac62d935e360d1cc9b1a32d2b6d95ea6b3f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Jan 2021 19:25:00 +0000 Subject: [PATCH 01/12] Add more consistency tests --- tests/check_consistent.py | 67 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 7de4c616b2c2..2a8be332991f 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 -# For various reasons we need the contents of certain files to be +# For security (and simplicity) reasons, only a limited kind of files can be +# present in /stdlib and /stubs directories, see README for detail. Here we +# verify these constraints. + +# In addition, for various reasons we need the contents of certain files to be # duplicated in two places, for example stdlib/@python2/builtins.pyi and # stdlib/@python2/__builtin__.pyi must be identical. In the past we used # symlinks but that doesn't always work on Windows, so now you must @@ -16,7 +20,62 @@ ] -def main(): +def assert_stubs_only(directory): + """Check that given directory contains only valid stub files.""" + assert directory.split(os.sep)[-1].isidentifier() + for root, dirs, files in os.walk(directory): + for file in files: + name, ext = os.path.splitext(file) + assert name.isidentifier(), "Files must be valid modules" + assert ext == "pyi", "Only stub flies allowed" + for subdir in dirs: + assert subdir.isidentifier(), "Directories must be valid packages" + + +def check_stdlib(): + for entry in os.listdir("stubs"): + if os.path.isfile(entry): + name, ext = os.path.splitext(entry) + if ext != "pyi": + assert entry == "VERSIONS", "Unexpected file in stdlib root" + assert name.isidentifier(), "Bad file name in stdlib" + else: + if entry == "@python2": + continue + assert_stubs_only(entry) + for entry in os.listdir("stubs/@python2"): + if os.path.isfile(entry): + name, ext = os.path.splitext(entry) + assert name.isidentifier(), "Bad file name in stdlib" + assert ext == "pyi", "Unexpected file in stdlib/@python2 root" + else: + assert_stubs_only(entry) + + +def check_stubs(): + for distribution in os.listdir("stubs"): + assert not os.path.isfile(distribution), "Only directories allowed in stubs" + for entry in os.listdir(os.path.join("stubs", distribution)): + if os.path.isfile(entry): + name, ext = os.path.splitext(entry) + if ext != "pyi": + assert entry in {"METADATA.toml", "README", "README.md", "README.rst"} + else: + assert name.isidentifier(), "Bad file name in stubs" + else: + if entry == "@python2": + continue + assert_stubs_only(entry) + for entry in os.listdir(os.path.join("stubs", distribution)): + if os.path.isfile(entry): + name, ext = os.path.splitext(entry) + assert name.isidentifier(), "Bad file name in stubs" + assert ext == "pyi", "Unexpected file in @python2 stubs" + else: + assert_stubs_only(entry) + + +def check_same_files(): files = [os.path.join(root, file) for root, dir, files in os.walk(".") for file in files] no_symlink = "You cannot use symlinks in typeshed, please copy {} to its link." for file in files: @@ -35,4 +94,6 @@ def main(): if __name__ == "__main__": - main() + check_stdlib() + check_stubs() + check_same_files() From 608f19fbfd3ac7b6e0c7a845b90f86d42f798916 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Jan 2021 20:05:06 +0000 Subject: [PATCH 02/12] Typos and fixes --- stubs/click/{click => }/README.md | 0 tests/check_consistent.py | 46 ++++++++++++++++--------------- 2 files changed, 24 insertions(+), 22 deletions(-) rename stubs/click/{click => }/README.md (100%) diff --git a/stubs/click/click/README.md b/stubs/click/README.md similarity index 100% rename from stubs/click/click/README.md rename to stubs/click/README.md diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 2a8be332991f..2d2ddc4500e5 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -22,57 +22,59 @@ def assert_stubs_only(directory): """Check that given directory contains only valid stub files.""" - assert directory.split(os.sep)[-1].isidentifier() + top = directory.split(os.sep)[-1] + assert top.isidentifier(), "Bad directory name: {}".format(top) for root, dirs, files in os.walk(directory): for file in files: name, ext = os.path.splitext(file) assert name.isidentifier(), "Files must be valid modules" - assert ext == "pyi", "Only stub flies allowed" + assert ext == ".pyi", "Only stub flies allowed. Got: {} in {}".format(file, directory) for subdir in dirs: assert subdir.isidentifier(), "Directories must be valid packages" def check_stdlib(): - for entry in os.listdir("stubs"): - if os.path.isfile(entry): + for entry in os.listdir("stdlib"): + if os.path.isfile(os.path.join("stdlib", entry)): name, ext = os.path.splitext(entry) - if ext != "pyi": - assert entry == "VERSIONS", "Unexpected file in stdlib root" + if ext != ".pyi": + assert entry == "VERSIONS", "Unexpected file in stdlib root: {}".format(entry) assert name.isidentifier(), "Bad file name in stdlib" else: if entry == "@python2": continue - assert_stubs_only(entry) - for entry in os.listdir("stubs/@python2"): - if os.path.isfile(entry): + assert_stubs_only(os.path.join("stdlib", entry)) + for entry in os.listdir("stdlib/@python2"): + if os.path.isfile(os.path.join("stdlib/@python2", entry)): name, ext = os.path.splitext(entry) assert name.isidentifier(), "Bad file name in stdlib" - assert ext == "pyi", "Unexpected file in stdlib/@python2 root" + assert ext == ".pyi", "Unexpected file in stdlib/@python2 root" else: - assert_stubs_only(entry) + assert_stubs_only(os.path.join("stdlib/@python2", entry)) def check_stubs(): for distribution in os.listdir("stubs"): assert not os.path.isfile(distribution), "Only directories allowed in stubs" for entry in os.listdir(os.path.join("stubs", distribution)): - if os.path.isfile(entry): + if os.path.isfile(os.path.join("stubs", distribution, entry)): name, ext = os.path.splitext(entry) - if ext != "pyi": - assert entry in {"METADATA.toml", "README", "README.md", "README.rst"} + if ext != ".pyi": + assert entry in {"METADATA.toml", "README", "README.md", "README.rst"}, entry else: assert name.isidentifier(), "Bad file name in stubs" else: if entry == "@python2": continue - assert_stubs_only(entry) - for entry in os.listdir(os.path.join("stubs", distribution)): - if os.path.isfile(entry): - name, ext = os.path.splitext(entry) - assert name.isidentifier(), "Bad file name in stubs" - assert ext == "pyi", "Unexpected file in @python2 stubs" - else: - assert_stubs_only(entry) + assert_stubs_only(os.path.join("stubs", distribution, entry)) + if os.path.isdir(os.path.join("stubs", distribution, "@python2")): + for entry in os.listdir(os.path.join("stubs", distribution, "@python2")): + if os.path.isfile(os.path.join("stubs", distribution, "@python2", entry)): + name, ext = os.path.splitext(entry) + assert name.isidentifier(), "Bad file name in stubs" + assert ext == ".pyi", "Unexpected file in @python2 stubs" + else: + assert_stubs_only(os.path.join("stubs", distribution, "@python2", entry)) def check_same_files(): From 6b393b53905953bb783494a9d0a322162f9a196c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 28 Jan 2021 20:07:59 +0000 Subject: [PATCH 03/12] Fix lint --- tests/check_consistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 2d2ddc4500e5..62d2ab06a8d0 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -24,7 +24,7 @@ def assert_stubs_only(directory): """Check that given directory contains only valid stub files.""" top = directory.split(os.sep)[-1] assert top.isidentifier(), "Bad directory name: {}".format(top) - for root, dirs, files in os.walk(directory): + for _, dirs, files in os.walk(directory): for file in files: name, ext = os.path.splitext(file) assert name.isidentifier(), "Files must be valid modules" From 2f2478fa405cbfeb832379297c616888c4b61d8c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 15:28:16 +0000 Subject: [PATCH 04/12] Add check for VERSIONS --- tests/check_consistent.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 62d2ab06a8d0..52180f8b7a65 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -95,7 +95,44 @@ def check_same_files(): ) +def check_versions(): + versions = {} + with open("stdlib/VERSIONS") as f: + data = f.read().splitlines() + for line in data: + if not line or line.lstrip().startswith("#"): + continue + assert ": " in line, f"Bad line in VERSIONS: {line}" + module, version = line.split(": ") + msg = f"Unsupported Python version{version}" + assert version.count(".") == 1, msg + major, minor = version.split(".") + assert major in {"2", "3"}, msg + assert minor.isdigit(), msg + assert module not in versions, f"Duplicate module {module} in VERSIONS" + modules = set() + for entry in os.listdir("stdlib"): + if entry == "@python2": + continue + if os.path.isfile(os.path.join("stdlib", entry)): + mod, _ = os.path.splitext(entry) + modules.add(mod) + else: + modules.add(entry) + for entry in os.listdir("stdlib/@python2"): + if os.path.isfile(os.path.join("stdlib/@python2", entry)): + mod, _ = os.path.splitext(entry) + modules.add(mod) + else: + modules.add(entry) + extra = modules - set(versions) + assert not extra, f"Modules not in versions: {extra}" + extra = set(versions) - modules + assert not extra, f"Versions not in modules: {extra}" + + if __name__ == "__main__": check_stdlib() + check_versions() check_stubs() check_same_files() From d6bd3deb54731f8ce07b8ca044269e09b77c2e2b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 15:58:59 +0000 Subject: [PATCH 05/12] Add check for METADATA.toml --- tests/check_consistent.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 52180f8b7a65..9d72459c6263 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -14,6 +14,8 @@ import filecmp import os +import toml + consistent_files = [ {"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"}, {"stdlib/threading.pyi", "stdlib/_dummy_threading.pyi"}, @@ -131,6 +133,29 @@ def check_versions(): assert not extra, f"Versions not in modules: {extra}" +def check_metadata(): + for distribution in os.listdir("stubs"): + with open(os.path.join("stubs", distribution, "METADATA.toml")) as f: + data = toml.loads(f.read()) + assert "version" in data, f"Missing version for {distribution}" + version = data["version"] + msg = f"Unsupported Python version{version}" + assert version.count(".") == 1, msg + major, minor = version.split(".") + assert major.isdigit() and minor.isdigit(), msg + for key in data: + assert key in { + "version", "python2", "python3", "requires" + }, f"Unexpected key {key} for {distribution}" + assert isinstance(data.get("python2", False), bool), f"Invalid python2 value for {distribution}" + assert isinstance(data.get("python3", True), bool), f"Invalid python3 value for {distribution}" + assert isinstance(data.get("requires", []), list), f"Invalid requires value for {distribution}" + for dep in data.get("requires", []): + # TODO: add more validation here. + assert isinstance(dep, str), f"Invalid dependency {dep} for {distribution}" + assert dep.startswith("types-"), f"Only stub dependencies supported, got {dep}" + + if __name__ == "__main__": check_stdlib() check_versions() From 0f16c2c052e95f77f6487dc8999cf412b65d16d6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:11:56 +0000 Subject: [PATCH 06/12] Fixes --- tests/check_consistent.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 9d72459c6263..4c2fa8235119 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -112,9 +112,10 @@ def check_versions(): assert major in {"2", "3"}, msg assert minor.isdigit(), msg assert module not in versions, f"Duplicate module {module} in VERSIONS" + versions[module] = (int(major), int(minor)) modules = set() for entry in os.listdir("stdlib"): - if entry == "@python2": + if entry == "@python2" or entry == "VERSIONS": continue if os.path.isfile(os.path.join("stdlib", entry)): mod, _ = os.path.splitext(entry) @@ -127,8 +128,9 @@ def check_versions(): modules.add(mod) else: modules.add(entry) - extra = modules - set(versions) - assert not extra, f"Modules not in versions: {extra}" + # TODO: fix and re-enable. + # extra = modules - set(versions) + # assert not extra, f"Modules not in versions: {extra}" extra = set(versions) - modules assert not extra, f"Versions not in modules: {extra}" From e1df4fdac2b44d0d08160321f0d03ac27a995605 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:13:27 +0000 Subject: [PATCH 07/12] Use f-strings --- tests/check_consistent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 4c2fa8235119..b525819bddd6 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -25,12 +25,12 @@ def assert_stubs_only(directory): """Check that given directory contains only valid stub files.""" top = directory.split(os.sep)[-1] - assert top.isidentifier(), "Bad directory name: {}".format(top) + assert top.isidentifier(), f"Bad directory name: {top}" for _, dirs, files in os.walk(directory): for file in files: name, ext = os.path.splitext(file) assert name.isidentifier(), "Files must be valid modules" - assert ext == ".pyi", "Only stub flies allowed. Got: {} in {}".format(file, directory) + assert ext == ".pyi", f"Only stub flies allowed. Got: {file} in {directory}" for subdir in dirs: assert subdir.isidentifier(), "Directories must be valid packages" @@ -40,7 +40,7 @@ def check_stdlib(): if os.path.isfile(os.path.join("stdlib", entry)): name, ext = os.path.splitext(entry) if ext != ".pyi": - assert entry == "VERSIONS", "Unexpected file in stdlib root: {}".format(entry) + assert entry == "VERSIONS", f"Unexpected file in stdlib root: {entry}" assert name.isidentifier(), "Bad file name in stdlib" else: if entry == "@python2": From dc5e17c5d1f559d4f584602928454cf377f65b56 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:17:08 +0000 Subject: [PATCH 08/12] Better error messages --- tests/check_consistent.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index b525819bddd6..93413e78dfe2 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -29,10 +29,10 @@ def assert_stubs_only(directory): for _, dirs, files in os.walk(directory): for file in files: name, ext = os.path.splitext(file) - assert name.isidentifier(), "Files must be valid modules" + assert name.isidentifier(), f"Files must be valid modules, got: {name}" assert ext == ".pyi", f"Only stub flies allowed. Got: {file} in {directory}" for subdir in dirs: - assert subdir.isidentifier(), "Directories must be valid packages" + assert subdir.isidentifier(), f"Directories must be valid packages, got: {subdir}" def check_stdlib(): @@ -57,14 +57,14 @@ def check_stdlib(): def check_stubs(): for distribution in os.listdir("stubs"): - assert not os.path.isfile(distribution), "Only directories allowed in stubs" + assert not os.path.isfile(distribution), f"Only directories allowed in stubs, got {distribution}" for entry in os.listdir(os.path.join("stubs", distribution)): if os.path.isfile(os.path.join("stubs", distribution, entry)): name, ext = os.path.splitext(entry) if ext != ".pyi": assert entry in {"METADATA.toml", "README", "README.md", "README.rst"}, entry else: - assert name.isidentifier(), "Bad file name in stubs" + assert name.isidentifier(), f"Bad file name '{entry}' in stubs" else: if entry == "@python2": continue @@ -73,8 +73,8 @@ def check_stubs(): for entry in os.listdir(os.path.join("stubs", distribution, "@python2")): if os.path.isfile(os.path.join("stubs", distribution, "@python2", entry)): name, ext = os.path.splitext(entry) - assert name.isidentifier(), "Bad file name in stubs" - assert ext == ".pyi", "Unexpected file in @python2 stubs" + assert name.isidentifier(), f"Bad file name '{entry}' in stubs" + assert ext == ".pyi", f"Unexpected file {entry} in @python2 stubs" else: assert_stubs_only(os.path.join("stubs", distribution, "@python2", entry)) From 9a5c68bd9673b2a9e74378f983a4ce0569580960 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:36:34 +0000 Subject: [PATCH 09/12] Fix dependency --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a0a4a98a4ec5..41e2e2163ff7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,9 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: ./tests/check_consistent.py + - run: | + pip install toml + ./tests/check_consistent.py flake8: name: Lint with flake8 From d2955976abf6b8ef7a1959abf6d0d070bd1bb770 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:50:56 +0000 Subject: [PATCH 10/12] Only check versions for 3 and 2and3 modules --- tests/check_consistent.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 93413e78dfe2..242e516d10d0 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -122,15 +122,8 @@ def check_versions(): modules.add(mod) else: modules.add(entry) - for entry in os.listdir("stdlib/@python2"): - if os.path.isfile(os.path.join("stdlib/@python2", entry)): - mod, _ = os.path.splitext(entry) - modules.add(mod) - else: - modules.add(entry) - # TODO: fix and re-enable. - # extra = modules - set(versions) - # assert not extra, f"Modules not in versions: {extra}" + extra = modules - set(versions) + assert not extra, f"Modules not in versions: {extra}" extra = set(versions) - modules assert not extra, f"Versions not in modules: {extra}" From beccc25b76e5312df5a2d4b2855b61fe49030635 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:57:51 +0000 Subject: [PATCH 11/12] Actually run metadata tests --- tests/check_consistent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index 242e516d10d0..df0bc8b499a5 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -155,4 +155,5 @@ def check_metadata(): check_stdlib() check_versions() check_stubs() + check_metadata() check_same_files() From 3419ac38475600004fa1a1daee944570872123d0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 29 Jan 2021 16:59:19 +0000 Subject: [PATCH 12/12] Fix space --- tests/check_consistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check_consistent.py b/tests/check_consistent.py index df0bc8b499a5..e779238b440f 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -134,7 +134,7 @@ def check_metadata(): data = toml.loads(f.read()) assert "version" in data, f"Missing version for {distribution}" version = data["version"] - msg = f"Unsupported Python version{version}" + msg = f"Unsupported Python version {version}" assert version.count(".") == 1, msg major, minor = version.split(".") assert major.isdigit() and minor.isdigit(), msg