From ad7fbfa973610d7f6f9b537f1b29cf7a9f4e89d7 Mon Sep 17 00:00:00 2001 From: yushan Date: Thu, 10 Jul 2025 20:45:39 +0000 Subject: [PATCH 1/3] clean python imports --- gazelle/python/file_parser.go | 10 ++++++++++ gazelle/python/file_parser_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index 31fce02712..5240335486 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -144,6 +144,14 @@ func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { return Module{}, false } +// cleanImportString removes backslashes and all whitespace from the string. +func cleanImportString(s string) string { + s = strings.ReplaceAll(s, "\\", "") + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, "\n", "") + return s +} + // parseImportStatements parses a node for import statements, returning true if the node is // an import statement. It updates FileParser.output.Modules with the `module` that the // import represents. @@ -154,6 +162,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { if !ok { continue } + m.From = cleanImportString(m.From) m.Filepath = p.relFilepath m.TypeCheckingOnly = p.inTypeCheckingBlock if strings.HasPrefix(m.Name, ".") { @@ -163,6 +172,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } } else if node.Type() == sitterNodeTypeImportFromStatement { from := node.Child(1).Content(p.code) + from = cleanImportString(from) // If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1. // If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules. if from == "." { diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index f4db1a316b..3204923b88 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go @@ -291,3 +291,32 @@ def example_function(): } } } + +func TestParseImportStatements_MultilineWithBackslashAndWhitespace(t *testing.T) { + p := NewFileParser() + code := []byte(`from foo.bar.\ + baz import ( + Something, + AnotherThing +) +`) + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Should parse as: from foo.bar.baz import Something, AnotherThing + expected := []Module{ + { + Name: "foo.bar.baz.Something", + LineNumber: 3, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.bar.baz.AnotherThing", + LineNumber: 4, + Filepath: "test.py", + From: "foo.bar.baz", + }, + } + assert.Equal(t, expected, output.Modules) +} From 7f53d6c4726d872cac2b1ba5a3f32865ddb5fbeb Mon Sep 17 00:00:00 2001 From: yushan Date: Thu, 10 Jul 2025 22:30:22 +0000 Subject: [PATCH 2/3] Update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f02c8bbb4..1b4a211d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (gazelle) Cleanup multi-line python imports modules {#v0-0-0-fixed} ### Fixed From f602ffa884c7bf72d80628610342085321818520 Mon Sep 17 00:00:00 2001 From: yushan Date: Mon, 14 Jul 2025 22:19:42 +0000 Subject: [PATCH 3/3] address comments --- CHANGELOG.md | 2 +- gazelle/python/file_parser.go | 4 + gazelle/python/file_parser_test.go | 105 ++++++++++++++---- .../import_nested_var/__init__.py | 6 +- 4 files changed, 94 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4a211d6a..1881acfd25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,6 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) -* (gazelle) Cleanup multi-line python imports modules {#v0-0-0-fixed} ### Fixed @@ -87,6 +86,7 @@ END_UNRELEASED_TEMPLATE ({gh-issue}`3043`). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. {#v0-0-0-added} ### Added diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index 5240335486..e129337e11 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -146,9 +146,11 @@ func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { // cleanImportString removes backslashes and all whitespace from the string. func cleanImportString(s string) string { + s = strings.ReplaceAll(s, "\r\n", "") s = strings.ReplaceAll(s, "\\", "") s = strings.ReplaceAll(s, " ", "") s = strings.ReplaceAll(s, "\n", "") + s = strings.ReplaceAll(s, "\t", "") return s } @@ -163,6 +165,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { continue } m.From = cleanImportString(m.From) + m.Name = cleanImportString(m.Name) m.Filepath = p.relFilepath m.TypeCheckingOnly = p.inTypeCheckingBlock if strings.HasPrefix(m.Name, ".") { @@ -185,6 +188,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } m.Filepath = p.relFilepath m.From = from + m.Name = cleanImportString(m.Name) m.Name = fmt.Sprintf("%s.%s", from, m.Name) m.TypeCheckingOnly = p.inTypeCheckingBlock p.output.Modules = append(p.output.Modules, m) diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index 3204923b88..0a6fd1b4ab 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go @@ -293,30 +293,93 @@ def example_function(): } func TestParseImportStatements_MultilineWithBackslashAndWhitespace(t *testing.T) { - p := NewFileParser() - code := []byte(`from foo.bar.\ + t.Parallel() + t.Run("multiline from import", func(t *testing.T) { + p := NewFileParser() + code := []byte(`from foo.bar.\ baz import ( Something, AnotherThing ) + +from foo\ + .test import ( + Foo, + Bar +) `) - p.SetCodeAndFile(code, "", "test.py") - output, err := p.Parse(context.Background()) - assert.NoError(t, err) - // Should parse as: from foo.bar.baz import Something, AnotherThing - expected := []Module{ - { - Name: "foo.bar.baz.Something", - LineNumber: 3, - Filepath: "test.py", - From: "foo.bar.baz", - }, - { - Name: "foo.bar.baz.AnotherThing", - LineNumber: 4, - Filepath: "test.py", - From: "foo.bar.baz", - }, - } - assert.Equal(t, expected, output.Modules) + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz.Something", + LineNumber: 3, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.bar.baz.AnotherThing", + LineNumber: 4, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.test.Foo", + LineNumber: 9, + Filepath: "test.py", + From: "foo.test", + }, + { + Name: "foo.test.Bar", + LineNumber: 10, + Filepath: "test.py", + From: "foo.test", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) + t.Run("multiline import", func(t *testing.T) { + p := NewFileParser() + code := []byte(`import foo.bar.\ + baz +`) + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz", + LineNumber: 1, + Filepath: "test.py", + From: "", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) + t.Run("windows line endings", func(t *testing.T) { + p := NewFileParser() + code := []byte("from foo.bar.\r\n baz import (\r\n Something,\r\n AnotherThing\r\n)\r\n") + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz.Something", + LineNumber: 3, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.bar.baz.AnotherThing", + LineNumber: 4, + Filepath: "test.py", + From: "foo.bar.baz", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) } diff --git a/gazelle/python/testdata/from_imports/import_nested_var/__init__.py b/gazelle/python/testdata/from_imports/import_nested_var/__init__.py index d0f51c443c..20eda530e5 100644 --- a/gazelle/python/testdata/from_imports/import_nested_var/__init__.py +++ b/gazelle/python/testdata/from_imports/import_nested_var/__init__.py @@ -13,4 +13,8 @@ # limitations under the License. # baz is a variable in foo/bar/baz.py -from foo.bar.baz import baz +from foo\ + .bar.\ + baz import ( + baz + )