git url: support CSV form

CSV parameters can be now specified after "##" in a git URL.
e.g.,
https://github.com/user/repo.git##tag=mytag,branch=main,commit=cafebab,subdir=/dir

tag, branch, and commit are exclusive.
The follow-up commit will allow specifying tag and commit together.

Fix issue 4905, but "source" in the original proposal was renamed to "subdir".

The documents will be added in
https://github.com/docker/docs/blob/main/content/manuals/build/concepts/context.md#url-fragments

Signed-off-by: Akihiro Suda <[email protected]>

Author: AkihiroSuda

util/gitutil/git_ref.go

@@ -61,11 +61,14 @@ func ParseGitRef(ref string) (*GitRef, error) {
 		return nil, cerrdefs.ErrInvalidArgument
 	} else if strings.HasPrefix(ref, "github.com/") {
 		res.IndistinguishableFromLocal = true // Deprecated
-		remote = fromURL(&url.URL{
+		remote, err = fromURL(&url.URL{
 			Scheme: "https",
 			Host:   "github.com",
 			Path:   strings.TrimPrefix(ref, "github.com/"),
 		})
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		remote, err = ParseURL(ref)
 		if errors.Is(err, ErrUnknownProtocol) {

util/gitutil/git_url.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/moby/buildkit/util/sshutil"
 	"github.com/pkg/errors"
+	"github.com/tonistiigi/go-csvvalue"
 )
 
 const (
@@ -66,12 +67,59 @@ type GitURLFragment struct {
 
 // splitGitFragment splits a git URL fragment into its respective git
 // reference and subdirectory components.
-func splitGitFragment(fragment string) *GitURLFragment {
+func splitGitFragment(fragment string) (*GitURLFragment, error) {
+	if strings.HasPrefix(fragment, "#") {
+		// Double-hash in the unparsed URL.
+		// e.g., https://github.com/user/repo.git##tag=tag,subdir=/dir
+		return splitGitFragmentCSVForm(fragment)
+	}
+	// Single-hash in the unparsed URL.
+	// e.g., https://github.com/user/repo.git#branch_or_tag_or_commit:dir
 	if fragment == "" {
-		return nil
+		return nil, nil
 	}
 	ref, subdir, _ := strings.Cut(fragment, ":")
-	return &GitURLFragment{Ref: ref, Subdir: subdir}
+	return &GitURLFragment{Ref: ref, Subdir: subdir}, nil
+}
+
+func splitGitFragmentCSVForm(fragment string) (*GitURLFragment, error) {
+	fragment = strings.TrimPrefix(fragment, "#")
+	if fragment == "" {
+		return nil, nil
+	}
+	fields, err := csvvalue.Fields(fragment, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse CSV %q", fragment)
+	}
+
+	res := &GitURLFragment{}
+	refs := make(map[string]string)
+	for _, field := range fields {
+		key, value, ok := strings.Cut(field, "=")
+		if !ok {
+			return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
+		}
+		key = strings.ToLower(key)
+		switch key {
+		case "tag", "branch", "commit":
+			refs[key] = value
+		case "subdir":
+			res.Subdir = value
+		default:
+			return nil, errors.Errorf("unexpected key '%s' in '%s' (supported keys: tag, branch, commit, subdir)", key, field)
+		}
+	}
+	if len(refs) > 0 {
+		if len(refs) > 1 {
+			// TODO: allow specifying tag and commit together https://github.com/moby/buildkit/issues/5871
+			return nil, errors.New("tag, branch, and commit are exclusive")
+		}
+		for _, v := range refs {
+			res.Ref = v
+			break
+		}
+	}
+	return res, nil
 }
 
 // ParseURL parses a BuildKit-style Git URL (that may contain additional
@@ -86,11 +134,11 @@ func ParseURL(remote string) (*GitURL, error) {
 		if err != nil {
 			return nil, err
 		}
-		return fromURL(url), nil
+		return fromURL(url)
 	}
 
 	if url, err := sshutil.ParseSCPStyleURL(remote); err == nil {
-		return fromSCPStyleURL(url), nil
+		return fromSCPStyleURL(url)
 	}
 
 	return nil, ErrUnknownProtocol
@@ -105,28 +153,37 @@ func IsGitTransport(remote string) bool {
 	return sshutil.IsImplicitSSHTransport(remote)
 }
 
-func fromURL(url *url.URL) *GitURL {
+func fromURL(url *url.URL) (*GitURL, error) {
 	withoutFragment := *url
 	withoutFragment.Fragment = ""
-	return &GitURL{
+	fragment, err := splitGitFragment(url.Fragment)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse URL fragment %q", url.Fragment)
+	}
+	gitURL := &GitURL{
 		Scheme:   url.Scheme,
 		User:     url.User,
 		Host:     url.Host,
 		Path:     url.Path,
-		Fragment: splitGitFragment(url.Fragment),
+		Fragment: fragment,
 		Remote:   withoutFragment.String(),
 	}
+	return gitURL, nil
 }
 
-func fromSCPStyleURL(url *sshutil.SCPStyleURL) *GitURL {
+func fromSCPStyleURL(url *sshutil.SCPStyleURL) (*GitURL, error) {
 	withoutFragment := *url
 	withoutFragment.Fragment = ""
+	fragment, err := splitGitFragment(url.Fragment)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse URL fragment %q", url.Fragment)
+	}
 	return &GitURL{
 		Scheme:   SSHProtocol,
 		User:     url.User,
 		Host:     url.Host,
 		Path:     url.Path,
-		Fragment: splitGitFragment(url.Fragment),
+		Fragment: fragment,
 		Remote:   withoutFragment.String(),
-	}
+	}, nil
 }

util/gitutil/git_url_test.go

@@ -151,6 +151,46 @@ func TestParseURL(t *testing.T) {
 				Path:   "/moby/buildkit",
 			},
 		},
+		{
+			url: "https://github.com/moby/buildkit##tag=v1.0.0,subdir=/subdir",
+			result: GitURL{
+				Scheme:   HTTPSProtocol,
+				Host:     "github.com",
+				Path:     "/moby/buildkit",
+				Fragment: &GitURLFragment{Ref: "v1.0.0", Subdir: "/subdir"},
+			},
+		},
+		{
+			url: `https://github.com/moby/buildkit##"tag=tag,with,comma",subdir=/subdir`,
+			result: GitURL{
+				Scheme:   HTTPSProtocol,
+				Host:     "github.com",
+				Path:     "/moby/buildkit",
+				Fragment: &GitURLFragment{Ref: "tag,with,comma", Subdir: "/subdir"},
+			},
+		},
+		{
+			url: "https://github.com/moby/buildkit##branch=v1.0,subdir=subdir",
+			result: GitURL{
+				Scheme:   HTTPSProtocol,
+				Host:     "github.com",
+				Path:     "/moby/buildkit",
+				Fragment: &GitURLFragment{Ref: "v1.0", Subdir: "subdir"},
+			},
+		},
+		{
+			url: "https://github.com/moby/buildkit##commit=deadbeef,subdir=/subdir",
+			result: GitURL{
+				Scheme:   HTTPSProtocol,
+				Host:     "github.com",
+				Path:     "/moby/buildkit",
+				Fragment: &GitURLFragment{Ref: "deadbeef", Subdir: "/subdir"},
+			},
+		},
+		{
+			url: "https://github.com/moby/buildkit##tag=v1.0.0,branch=v1.0",
+			err: true, // tag and branch are exclusive
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.url, func(t *testing.T) {