fix(gazelle): Support parsing files that use Python3.12 PEP 695 (Type Parameter Syntax) by using dougthor42's fork of go-tree-sitter (#2496)

Replaces #2413.

Fixes #2396.

This updates the `go-tree-sitter` dependency to use my fork that
includes `BUILD.bazel` files. Specifically, the `BUILD.bazel` files in
the fork include references to top-level code like `array.h` which the
original Gazelle-generated files for `go-tree-sitter` were not able to
handle. I also include the test cases that @maffoo created in #2413 and
verified that they (a) fail before the fix and (b) pass after the fix.

The fork is: https://github.com/dougthor42/go-tree-sitter

The branch that includes all changes is:
https://github.com/dougthor42/go-tree-sitter/tree/for-rules-python-gazelle-plugin

A couple notes:
+ I have a PR open to get `go-tree-sitter` into BCR
[here](bazelbuild/bazel-central-registry#3366).
However:
1. I'm having trouble getting tests to pass and to get things running
locally to validate it
2. Using BCR would not fix things for people who still use WORKSPACE
(right?)
+ The fork is _mostly_ [autogenerated BUILD.bazel files from
gazelle](smacker/go-tree-sitter@cfa9bdf)
but also contains:
+ [manual updates so that build files reference the toplevel `array.h`
and other
files](smacker/go-tree-sitter@63f89cd)
+ [replace all `smacker` with `dougthor42` so that `go build`
works](smacker/go-tree-sitter@8a73cbd)
    + various other more minor things.
+ I was unable to get `go mod edit -replace` to work, so I've just
manually updated `go.mod` and whatnot everywhere. If someone with more
go knowledge has a suggestion I'm happy to hear it.

---------

Co-authored-by: Matthew Neeley <[email protected]>
Co-authored-by: Ignas Anikevicius <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -65,12 +65,16 @@ Unreleased changes template.
   available for all of the platforms and the sdist can be built. This fix is
   for both WORKSPACE and `bzlmod` setups.
   Fixes [#2450](https://github.com/bazelbuild/rules_python/issues/2450).
+* (gazelle) Gazelle will now correctly parse Python3.12 files that use [PEP 695 Type
+  Parameter Syntax][pep-695]. (#2396)
+
+[pep-695]: https://peps.python.org/pep-0695/
 
 {#v0-0-0-added}
 ### Added
 * (gazelle) Added `include_stub_packages`  flag to `modules_mapping`. When set to `True`, this
   automatically includes corresponding stub packages for third-party libraries
-  that are present and used (e.g., `boto3` → `boto3-stubs`), improving 
+  that are present and used (e.g., `boto3` → `boto3-stubs`), improving
   type-checking support.
 * (pypi) Freethreaded packages are now fully supported in the
   {obj}`experimental_index_url` usage or the regular `pip.parse` usage.
@@ -137,7 +141,7 @@ Other changes:
 * (repositories): Add libs/python3.lib and pythonXY.dll to the `libpython` target
   defined by a repository template. This enables stable ABI builds of Python extensions
   on Windows (by defining Py_LIMITED_API).
-* (rules) `py_test` and `py_binary` targets no longer incorrectly remove the 
+* (rules) `py_test` and `py_binary` targets no longer incorrectly remove the
   first `sys.path` entry when using {obj}`--bootstrap_impl=script`
 
 {#v1-0-0-added}

gazelle/MODULE.bazel

@@ -21,9 +21,9 @@ use_repo(
     go_deps,
     "com_github_bazelbuild_buildtools",
     "com_github_bmatcuk_doublestar_v4",
+    "com_github_dougthor42_go_tree_sitter",
     "com_github_emirpasic_gods",
     "com_github_ghodss_yaml",
-    "com_github_smacker_go_tree_sitter",
     "com_github_stretchr_testify",
     "in_gopkg_yaml_v2",
     "org_golang_x_sync",

gazelle/deps.bzl

@@ -186,10 +186,10 @@ def go_deps():
         version = "v0.0.0-20190812154241-14fe0d1b01d4",
     )
     go_repository(
-        name = "com_github_smacker_go_tree_sitter",
-        importpath = "github.com/smacker/go-tree-sitter",
-        sum = "h1:7QZKUmQfnxncZIJGyvX8M8YeMfn8kM10j3J/2KwVTN4=",
-        version = "v0.0.0-20240422154435-0628b34cbf9c",
+        name = "com_github_dougthor42_go_tree_sitter",
+        importpath = "github.com/dougthor42/go-tree-sitter",
+        sum = "h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ=",
+        version = "v0.0.0-20241210060307-2737e1d0de6b",
     )
     go_repository(
         name = "com_github_stretchr_objx",

gazelle/go.mod

@@ -7,9 +7,9 @@ require (
 	github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82
 	github.com/bazelbuild/rules_go v0.41.0
 	github.com/bmatcuk/doublestar/v4 v4.6.1
+	github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b
 	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0
-	github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/sync v0.2.0
 	gopkg.in/yaml.v2 v2.4.0

gazelle/go.sum

@@ -13,9 +13,10 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ=
+github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b/go.mod h1:87UkDyPt18bTH/FvinLc/kj587VNYOdRKZT1la4T8Hg=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -44,12 +45,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c h1:7QZKUmQfnxncZIJGyvX8M8YeMfn8kM10j3J/2KwVTN4=
-github.com/smacker/go-tree-sitter v0.0.0-20240422154435-0628b34cbf9c/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
@@ -105,7 +100,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

gazelle/python/BUILD.bazel

@@ -39,11 +39,11 @@ go_library(
         "@bazel_gazelle//rule:go_default_library",
         "@com_github_bazelbuild_buildtools//build:go_default_library",
         "@com_github_bmatcuk_doublestar_v4//:doublestar",
+        "@com_github_dougthor42_go_tree_sitter//:go-tree-sitter",
+        "@com_github_dougthor42_go_tree_sitter//python",
         "@com_github_emirpasic_gods//lists/singlylinkedlist",
         "@com_github_emirpasic_gods//sets/treeset",
         "@com_github_emirpasic_gods//utils",
-        "@com_github_smacker_go_tree_sitter//:go-tree-sitter",
-        "@com_github_smacker_go_tree_sitter//python",
         "@org_golang_x_sync//errgroup",
     ],
 )

gazelle/python/file_parser.go

@@ -22,8 +22,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	sitter "github.com/smacker/go-tree-sitter"
-	"github.com/smacker/go-tree-sitter/python"
+	sitter "github.com/dougthor42/go-tree-sitter"
+	"github.com/dougthor42/go-tree-sitter/python"
 )
 
 const (
@@ -115,10 +115,10 @@ func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool {
 				a, b = b, a
 			}
 			if a.Type() == sitterNodeTypeIdentifier && a.Content(p.code) == "__name__" &&
-				// at github.com/smacker/go-tree-sitter@latest (after v0.0.0-20240422154435-0628b34cbf9c we used)
+				// at github.com/dougthor42/go-tree-sitter@latest (after v0.0.0-20240422154435-0628b34cbf9c we used)
 				// "__main__" is the second child of b. But now, it isn't.
 				// we cannot use the latest go-tree-sitter because of the top level reference in scanner.c.
-				// https://github.com/smacker/go-tree-sitter/blob/04d6b33fe138a98075210f5b770482ded024dc0f/python/scanner.c#L1
+				// https://github.com/dougthor42/go-tree-sitter/blob/04d6b33fe138a98075210f5b770482ded024dc0f/python/scanner.c#L1
 				b.Type() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" {
 				return true
 			}

gazelle/python/testdata/py312_syntax/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_generation_mode file

gazelle/python/testdata/py312_syntax/BUILD.out

@@ -0,0 +1,16 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
+
+# gazelle:python_generation_mode file
+
+py_library(
+    name = "_other_module",
+    srcs = ["_other_module.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_binary(
+    name = "pep_695_type_parameter",
+    srcs = ["pep_695_type_parameter.py"],
+    visibility = ["//:__subpackages__"],
+    deps = [":_other_module"],
+)

gazelle/python/testdata/py312_syntax/README.md

@@ -0,0 +1,4 @@
+# py312 syntax
+
+This test case checks that we properly parse certain python 3.12 syntax, such
+as pep 695 type parameters, with go-tree-sitter.

gazelle/python/testdata/py312_syntax/WORKSPACE

@@ -0,0 +1 @@
+# This is a Bazel workspace for the Gazelle test data.

gazelle/python/testdata/py312_syntax/__init__.py

@@ -0,0 +1,22 @@
+def search_one_more_level[T](
+    graph: dict[T, set[T]], seen: set[T], routes: list[list[T]], target: T
+) -> list[T] | None:
+    """This function fails to parse with older versions of go-tree-sitter.
+
+    Args:
+        graph: The graph to search as input.
+        seen: The nodes that have been visited as input/output.
+        routes: The current routes in the breadth-first search as input/output.
+        target: The target to search in this extra search level.
+
+    Returns:
+        a route if it ends on the target, or None if no route reaches the
+        target.
+    """
+
+
+import _other_module
+
+
+if __name__ == "__main__":
+    pass

gazelle/python/testdata/py312_syntax/_other_module.py

@@ -0,0 +1 @@
+---