From cba8071e6b5182c61fbdb5fe484500a7456f181b Mon Sep 17 00:00:00 2001 From: Bago Amirbekian Date: Thu, 30 Nov 2023 17:10:20 -0800 Subject: [PATCH 1/9] skip sys.path dirs with perm error --- rope/contrib/autoimport/sqlite.py | 15 +++++++++++++-- ropetest/contrib/autoimporttest.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index f02d426ba..f0215731c 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -546,13 +546,24 @@ def filter_folders(folder: Path) -> bool: folder_paths = filter(filter_folders, folder_paths) # type:ignore return list(OrderedDict.fromkeys(folder_paths)) + def _safe_iterdir(self, folder: Path): + print(type(folder)) + dirs = folder.iterdir() + while True: + try: + yield next(dirs) + except PermissionError: + pass + except StopIteration: + break + def _get_available_packages(self) -> List[Package]: packages: List[Package] = [ Package(module, Source.BUILTIN, None, PackageType.BUILTIN) for module in sys.builtin_module_names ] for folder in self._get_python_folders(): - for package in folder.iterdir(): + for package in self._safe_iterdir(folder): package_tuple = get_package_tuple(package, self.project) if package_tuple is None: continue @@ -602,7 +613,7 @@ def _find_package_path(self, target_name: str) -> Optional[Package]: if target_name in sys.builtin_module_names: return Package(target_name, Source.BUILTIN, None, PackageType.BUILTIN) for folder in self._get_python_folders(): - for package in folder.iterdir(): + for package in self._safe_iterdir(folder): package_tuple = get_package_tuple(package, self.project) if package_tuple is None: continue diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 62b6db293..f2804c822 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -142,6 +142,22 @@ def test_generate_full_cache(self): for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) + def test_generate_full_cache(self): + """The single thread test takes much longer than the multithread test but is easier to debug""" + single_thread = False + self.importer.generate_modules_cache(single_thread=single_thread) + """ + Create a temporary directory and set permissions to 000 + """ + import tempfile, sys + with tempfile.TemporaryDirectory() as dir: + import os + os.chmod(dir, 0) + sys.path.append(dir) + self.importer.generate_modules_cache(single_thread=single_thread) + self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) + self.assertTrue(len(self.importer._dump_all()) > 0) + class AutoImportObservingTest(unittest.TestCase): def setUp(self): From bc6a0c39991b32bdfce0809a8b793b253427c453 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 21 Dec 2023 16:19:27 +1100 Subject: [PATCH 2/9] Remove stray print statement --- rope/contrib/autoimport/sqlite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index f0215731c..1755f2f2b 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -547,7 +547,6 @@ def filter_folders(folder: Path) -> bool: return list(OrderedDict.fromkeys(folder_paths)) def _safe_iterdir(self, folder: Path): - print(type(folder)) dirs = folder.iterdir() while True: try: From 5ce1201844e6da82a06305afa8af96d70fed0d4f Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 21 Dec 2023 16:25:28 +1100 Subject: [PATCH 3/9] These doesn't seem to be docstring, they should just be regular comment --- ropetest/contrib/autoimporttest.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index f2804c822..43173f406 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -143,12 +143,10 @@ def test_generate_full_cache(self): self.assertTrue(len(table) > 0) def test_generate_full_cache(self): - """The single thread test takes much longer than the multithread test but is easier to debug""" - single_thread = False - self.importer.generate_modules_cache(single_thread=single_thread) - """ - Create a temporary directory and set permissions to 000 - """ + # The single thread test takes much longer than the multithread test but is easier to debug + self.importer.generate_modules_cache(single_thread=False) + + # Create a temporary directory and set permissions to 000 import tempfile, sys with tempfile.TemporaryDirectory() as dir: import os From 82d498e6c3557d0817fc5d9eb31a1ddd2c038831 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Thu, 21 Dec 2023 16:30:36 +1100 Subject: [PATCH 4/9] Restore the single_thread variable --- ropetest/contrib/autoimporttest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 43173f406..9e93a6e88 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -144,7 +144,8 @@ def test_generate_full_cache(self): def test_generate_full_cache(self): # The single thread test takes much longer than the multithread test but is easier to debug - self.importer.generate_modules_cache(single_thread=False) + single_thread = False + self.importer.generate_modules_cache(single_thread=single_thread) # Create a temporary directory and set permissions to 000 import tempfile, sys From a3b64ae457d94bf994f6906cda045cd89c2e8e53 Mon Sep 17 00:00:00 2001 From: Bago Amirbekian Date: Thu, 21 Dec 2023 11:32:30 -0800 Subject: [PATCH 5/9] use python_paths instead of sys.path for perms tests --- rope/contrib/autoimport/sqlite.py | 1 - ropetest/contrib/autoimporttest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index f0215731c..1755f2f2b 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -547,7 +547,6 @@ def filter_folders(folder: Path) -> bool: return list(OrderedDict.fromkeys(folder_paths)) def _safe_iterdir(self, folder: Path): - print(type(folder)) dirs = folder.iterdir() while True: try: diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index f2804c822..306a17c40 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -153,7 +153,7 @@ def test_generate_full_cache(self): with tempfile.TemporaryDirectory() as dir: import os os.chmod(dir, 0) - sys.path.append(dir) + self.importer.project.prefs.python_path = [dir] self.importer.generate_modules_cache(single_thread=single_thread) self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) self.assertTrue(len(self.importer._dump_all()) > 0) From c14b27385636a46b0cf5d81320f1dcfffae636d2 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Wed, 3 Jan 2024 11:59:12 +1100 Subject: [PATCH 6/9] Fix test name which duplicates existing test name --- ropetest/contrib/autoimporttest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 8b7e43fe1..52207ddc0 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -142,7 +142,7 @@ def test_generate_full_cache(self): for table in self.importer._dump_all(): self.assertTrue(len(table) > 0) - def test_generate_full_cache(self): + def test_skipping_directories_not_accessible_because_of_permission_error(self): # The single thread test takes much longer than the multithread test but is easier to debug single_thread = False self.importer.generate_modules_cache(single_thread=single_thread) From 351408cdbaf03fca24d19189f94de7abd44295b8 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Wed, 3 Jan 2024 11:59:35 +1100 Subject: [PATCH 7/9] Minor readability fixes - comment is not a docstring - use octal syntax for chmod --- ropetest/contrib/autoimporttest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 52207ddc0..1e543101b 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -134,7 +134,7 @@ def test_search(self): self.assertIn(import_statement, self.importer.search("D")) def test_generate_full_cache(self): - """The single thread test takes much longer than the multithread test but is easier to debug""" + # The single thread test takes much longer than the multithread test but is easier to debug single_thread = False self.importer.generate_modules_cache(single_thread=single_thread) self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) @@ -151,7 +151,7 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self): import tempfile, sys with tempfile.TemporaryDirectory() as dir: import os - os.chmod(dir, 0) + os.chmod(dir, 0o000) self.importer.project.prefs.python_path = [dir] self.importer.generate_modules_cache(single_thread=single_thread) self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) From 44236e381fc37ae2eb33e5d38d558a5056bf6949 Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Wed, 3 Jan 2024 12:02:07 +1100 Subject: [PATCH 8/9] Use assertGreater() instead of assertTrue(... > ...) --- ropetest/contrib/autoimporttest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ropetest/contrib/autoimporttest.py b/ropetest/contrib/autoimporttest.py index 1e543101b..94bb033bb 100644 --- a/ropetest/contrib/autoimporttest.py +++ b/ropetest/contrib/autoimporttest.py @@ -138,9 +138,9 @@ def test_generate_full_cache(self): single_thread = False self.importer.generate_modules_cache(single_thread=single_thread) self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) - self.assertTrue(len(self.importer._dump_all()) > 0) + self.assertGreater(len(self.importer._dump_all()), 0) for table in self.importer._dump_all(): - self.assertTrue(len(table) > 0) + self.assertGreater(len(table), 0) def test_skipping_directories_not_accessible_because_of_permission_error(self): # The single thread test takes much longer than the multithread test but is easier to debug @@ -155,7 +155,7 @@ def test_skipping_directories_not_accessible_because_of_permission_error(self): self.importer.project.prefs.python_path = [dir] self.importer.generate_modules_cache(single_thread=single_thread) self.assertIn(("from typing import Dict", "Dict"), self.importer.search("Dict")) - self.assertTrue(len(self.importer._dump_all()) > 0) + self.assertGreater(len(self.importer._dump_all()), 0) class AutoImportObservingTest(unittest.TestCase): From c946a9cf1355bef5109442242d7e88c2d403c34c Mon Sep 17 00:00:00 2001 From: Lie Ryan Date: Wed, 3 Jan 2024 12:05:46 +1100 Subject: [PATCH 9/9] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87f788dd3..337fabcd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # **Upcoming release** +- #733 skip directories with perm error when building autoimport index (@MrBago) + # Release 1.11.0 - #710, #561 Implement `except*` syntax (@lieryan)