feat(cli): add support for caching using watchman (#8398)

This adds a watchman based implementation of the cache used by
`configure`. When using watchman caching we can invalidate the cache
based on the watchman `GetDiff` since the last cache was written,
instead of invalidating the cache based on the file content.

With one large repo `configure` is 50+s, file caching 20s, watchman
caching gets it ~8s.

---

### Changes are visible to end-users: yes

- Searched for relevant documentation and updated as needed: yes
- Breaking change (forces users to change their own code or config): no
- Suggested release notes appear below: yes

Support `configure` use of `watchman` for file caching.

Set the `ASPECT_CONFIGURE_CACHE` env var to location to read+persist the
cache, pass the `--watchman` flag to `configure`. For example:

```
ASPECT_CONFIGURE_CACHE=configure.cache
aspect configure --watchman
```

NOTE: `ASPECT_CONFIGURE_CACHE` without `--watchman` uses basic file
caching and the file format is not compatible, different cache files
must be used for watchman vs disk based caching.

### Test plan

- Manual testing; run `configure` with `ASPECT_CONFIGURE_CACHE` +
`--watchman`

GitOrigin-RevId: 83385109e8894f9d96996ebe817a3d650618bf1c

Author: jbedard

cmd/aspect/configure/configure.go

@@ -38,7 +38,7 @@ func NewDefaultCmd() *cobra.Command {
 func NewCmd(streams ioutils.Streams) *cobra.Command {
 	return NewCmdWithConfigure(streams, configure.New(streams))
 }
-func NewCmdWithConfigure(streams ioutils.Streams, v *configure.Configure) *cobra.Command {
+func NewCmdWithConfigure(streams ioutils.Streams, v configure.ConfigureRunner) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "configure",
 		Short: "Auto-configure Bazel by updating BUILD files",
@@ -91,14 +91,14 @@ Run 'aspect help directives' or see https://docs.aspect.build/cli/help/directive
 	return cmd
 }
 
-func run(streams ioutils.Streams, v *configure.Configure, mode string, exclude []string, args []string) error {
+func run(streams ioutils.Streams, v configure.ConfigureRunner, mode string, exclude []string, args []string) error {
 	if buildinfo.Current().OpenSource {
 		if configurePlugins := viper.GetStringSlice("configure.plugins"); len(configurePlugins) > 0 {
 			fmt.Fprintln(streams.Stderr, "WARNING: Aspect CLI configure.plugins are not supported in Aspect OSS CLI.")
 		}
 	}
 
-	err := v.Run(mode, exclude, args)
+	err := v.Generate(mode, exclude, args)
 	if aspectError, isAError := err.(*aspecterrors.ExitError); isAError && aspectError.ExitCode == aspecterrors.ConfigureNoConfig {
 		fmt.Fprintln(streams.Stderr, `No languages enabled for BUILD file generation.
 
@@ -117,7 +117,7 @@ configure:
 	return err
 }
 
-func addCliEnabledLanguages(c *configure.Configure) {
+func addCliEnabledLanguages(c configure.ConfigureRunner) {
 	// Order matters for gazelle languages. Proto should be run before golang.
 	viper.SetDefault("configure.languages.protobuf", false)
 	if viper.GetBool("configure.languages.protobuf") {

gazelle/common/cache/cache.go

@@ -1,10 +1,14 @@
 package cache
 
+import "github.com/bazelbuild/bazel-gazelle/config"
+
 type Cache interface {
 	/** Load a key from the cache. */
 	Load(key string) (any, bool)
 	/** Store a key+value in the ache */
 	Store(key string, value any)
+	/** Store a key+value in the cache, or return the existing value */
+	LoadOrStore(key string, value any) (any, bool)
 
 	/** Persist any changes to the cache */
 	Persist()
@@ -20,5 +24,9 @@ type Cache interface {
 	 * The path 'root' is not part of the cache key, but is used to resolve
 	 * relative paths in the cache.
 	 */
-	LoadAndStoreFile(root, path string, loader func(path string, content []byte) (any, error)) (any, bool, error)
+	LoadOrStoreFile(root, path, key string, loader FileCompute) (any, bool, error)
 }
+
+type FileCompute = func(path string, content []byte) (any, error)
+
+type CacheFactory = func(c *config.Config) Cache

gazelle/common/cache/configurer.go

@@ -12,15 +12,27 @@ import (
 const gazelleExtensionKey = "__aspect:cache"
 
 func NewConfigurer() config.Configurer {
-	return &cacheConfigurer{}
+	return &cacheConfigurer{
+		cache: noop,
+	}
 }
 
 // Fetch the shared cache for a given config
-func Get[T any](config *config.Config) Cache {
+func Get(config *config.Config) Cache {
 	if v, ok := config.Exts[gazelleExtensionKey]; ok {
 		return v.(Cache)
 	}
-	return nil
+	return noop
+}
+
+var cacheFactory CacheFactory
+
+func init() {
+	if diskCachePath := os.Getenv("ASPECT_CONFIGURE_CACHE"); diskCachePath != "" {
+		cacheFactory = func(c *config.Config) Cache {
+			return NewDiskCache(diskCachePath)
+		}
+	}
 }
 
 var _ config.Configurer = (*cacheConfigurer)(nil)
@@ -30,22 +42,26 @@ type cacheConfigurer struct {
 	cache Cache
 }
 
+func SetCacheFactory(c CacheFactory) {
+	cacheFactory = c
+}
+
 // Load + store the cache
-func (cc *cacheConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {
-	if diskCachePath := os.Getenv("ASPECT_CONFIGURE_CACHE"); diskCachePath != "" {
-		cc.cache = NewDiskCache(diskCachePath)
+func (cc *cacheConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
+	if cacheFactory == nil {
+		cc.cache = noop
 	} else {
-		cc.cache = Noop()
+		cc.cache = cacheFactory(c)
 	}
-
 	c.Exts[gazelleExtensionKey] = cc.cache
+	return nil
 }
 
 // Persist the cache
 func (cc *cacheConfigurer) DoneGeneratingRules() {
 	cc.cache.Persist()
 }
 
-func (*cacheConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error  { return nil }
-func (*cacheConfigurer) KnownDirectives() []string                            { return nil }
-func (*cacheConfigurer) Configure(c *config.Config, rel string, f *rule.File) {}
+func (cc *cacheConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *config.Config) {}
+func (*cacheConfigurer) KnownDirectives() []string                                       { return nil }
+func (*cacheConfigurer) Configure(c *config.Config, rel string, f *rule.File)            {}

gazelle/common/cache/disk.go

@@ -38,22 +38,13 @@ type diskCache struct {
 	new  *sync.Map
 }
 
-func computeCacheKey(content []byte) (string, bool) {
+func computeCacheKey(content []byte, key string) string {
 	cacheDigest := crypto.MD5.New()
-
 	if buildinfo.IsStamped() {
-		if _, err := cacheDigest.Write([]byte(buildinfo.GitCommit)); err != nil {
-			BazelLog.Errorf("Failed to write GitCommit to cache digest: %v", err)
-			return "", false
-		}
+		cacheDigest.Write([]byte(buildinfo.GitCommit))
 	}
-
-	if _, err := cacheDigest.Write(content); err != nil {
-		BazelLog.Errorf("Failed to write source to cache digest: %v", err)
-		return "", false
-	}
-
-	return hex.EncodeToString(cacheDigest.Sum(nil)), true
+	cacheDigest.Write([]byte(key))
+	return hex.EncodeToString(cacheDigest.Sum(content))
 }
 
 func (c *diskCache) read() {
@@ -91,46 +82,50 @@ func (c *diskCache) write() {
 }
 
 func (c *diskCache) Load(key string) (any, bool) {
+	// Already written to new cache.
 	if v, found := c.new.Load(key); found {
 		return v, true
 	}
 
+	// Exists in old cache and can transfer to new.
 	if v, ok := c.old[key]; ok {
-		c.new.LoadOrStore(key, v)
+		v, _ = c.LoadOrStore(key, v)
 		return v, true
 	}
 
+	// Cache miss
 	return nil, false
 }
 
 func (c *diskCache) Store(key string, value any) {
 	c.new.Store(key, value)
 }
 
-func (c *diskCache) LoadAndStoreFile(root, p string, loader func(path string, content []byte) (any, error)) (any, bool, error) {
+func (c *diskCache) LoadOrStore(key string, value any) (any, bool) {
+	return c.new.LoadOrStore(key, value)
+}
+
+func (c *diskCache) LoadOrStoreFile(root, p, key string, loader FileCompute) (any, bool, error) {
 	content, err := os.ReadFile(path.Join(root, p))
 	if err != nil {
 		return nil, false, err
 	}
 
-	if parserCacheKey, parsingCacheable := computeCacheKey(content); parsingCacheable {
-		// Try loading from the cache.
-		if v, found := c.Load(parserCacheKey); found {
-			return v, true, nil
-		}
+	// Include the file content in the cache key
+	key = computeCacheKey(content, key)
 
-		// Compute and persist in cache.
-		v, err := loader(p, content)
+	// Try loading from the cache.
+	v, found := c.Load(key)
+
+	// Compute and persist in cache.
+	if !found {
+		v, err = loader(p, content)
 		if err != nil {
-			c.Store(parserCacheKey, v)
+			v, found = c.LoadOrStore(key, v)
 		}
-
-		return v, false, err
 	}
 
-	// Not cacheable, simply recompute each time.
-	v, err := loader(p, content)
-	return v, false, err
+	return v, found, err
 }
 
 func (c *diskCache) Persist() {

gazelle/common/cache/noop.go

@@ -5,17 +5,18 @@ import (
 	"path"
 )
 
-func Noop() Cache {
-	return &noopCache{}
-}
-
 var _ Cache = (*noopCache)(nil)
 
+var noop Cache = &noopCache{}
+
 type noopCache struct{}
 
 func (c *noopCache) Load(key string) (any, bool) { return nil, false }
 func (c *noopCache) Store(key string, value any) {}
-func (c *noopCache) LoadAndStoreFile(root, p string, loader func(p string, content []byte) (any, error)) (any, bool, error) {
+func (c *noopCache) LoadOrStore(key string, value any) (any, bool) {
+	return value, false
+}
+func (c *noopCache) LoadOrStoreFile(root, p, key string, loader FileCompute) (any, bool, error) {
 	content, err := os.ReadFile(path.Join(root, p))
 	if err != nil {
 		return nil, false, err

gazelle/js/generate.go

@@ -832,16 +832,18 @@ func (ts *typeScriptLang) collectImports(cfg *JsGazelleConfig, config *config.Co
 func parseSourceFile(config *config.Config, rootDir, filePath string) (parser.ParseResult, error) {
 	BazelLog.Tracef("ParseImports(%s): %s", LanguageName, filePath)
 
-	parserCache := cache.Get[parser.ParseResult](config)
-	if parserCache == nil {
-		parserCache = cache.Noop()
-	}
+	parserCache := cache.Get(config)
 
-	r, _, err := parserCache.LoadAndStoreFile(rootDir, filePath, func(filePath string, content []byte) (any, error) {
+	var p parser.ParseResult
+	r, _, err := parserCache.LoadOrStoreFile(rootDir, filePath, "js.ParseSource", func(filePath string, content []byte) (any, error) {
 		return parser.ParseSource(filePath, content)
 	})
 
-	return r.(parser.ParseResult), err
+	if r != nil {
+		p = r.(parser.ParseResult)
+	}
+
+	return p, err
 }
 
 func init() {

pkg/aspect/configure/configure.go

@@ -35,13 +35,21 @@ import (
 	"golang.org/x/term"
 )
 
+type ConfigureRunner interface {
+	AddLanguage(lang ConfigureLanguage)
+	AddLanguageFactory(lang string, langFactory func() language.Language)
+	Generate(mode ConfigureMode, excludes []string, args []string) error
+}
+
 type Configure struct {
 	ioutils.Streams
 
 	languageKeys []string
 	languages    []func() language.Language
 }
 
+var _ ConfigureRunner = (*Configure)(nil)
+
 // Builtin Gazelle languages
 type ConfigureLanguage = string
 
@@ -117,7 +125,7 @@ func (c *Configure) AddLanguage(lang ConfigureLanguage) {
 	}
 }
 
-func (runner *Configure) Run(mode ConfigureMode, excludes []string, args []string) error {
+func (runner *Configure) Generate(mode ConfigureMode, excludes []string, args []string) error {
 	if len(runner.languageKeys) == 0 {
 		return &aspecterrors.ExitError{
 			ExitCode: aspecterrors.ConfigureNoConfig,