Merge pull request moby#32993 from cyli/root-rotation-cli

API changes to rotate swarm root CA

Author: aaronlehmann

api/swagger.yaml

@@ -1886,6 +1886,14 @@ definitions:
                 CACert:
                   description: "The root CA certificate (in PEM format) this external CA uses to issue TLS certificates (assumed to be to the current swarm root CA certificate if not provided)."
                   type: "string"
+          SigningCACert:
+            description: "The desired signing CA certificate for all swarm node TLS leaf certificates, in PEM format."
+            type: "string"
+          SigningCAKey:
+            description: "The desired signing CA key for all swarm node TLS leaf certificates, in PEM format."
+            type: "string"
+          ForceRotate:
+            description: "An integer whose purpose is to force swarm to generate a new signing CA certificate and key, if none have been specified in `SigningCACert` and `SigningCAKey`"
       EncryptionConfig:
         description: "Parameters related to encryption-at-rest."
         type: "object"

api/types/swarm/swarm.go

@@ -109,6 +109,16 @@ type CAConfig struct {
 	// ExternalCAs is a list of CAs to which a manager node will make
 	// certificate signing requests for node certificates.
 	ExternalCAs []*ExternalCA `json:",omitempty"`
+
+	// SigningCACert and SigningCAKey specify the desired signing root CA and
+	// root CA key for the swarm.  When inspecting the cluster, the key will
+	// be redacted.
+	SigningCACert string `json:",omitempty"`
+	SigningCAKey  string `json:",omitempty"`
+
+	// If this value changes, and there is no specified signing cert and key,
+	// then the swarm is forced to generate a new root certificate ane key.
+	ForceRotate uint64 `json:",omitempty"`
 }
 
 // ExternalCAProtocol represents type of external CA.

daemon/cluster/convert/swarm.go

@@ -30,6 +30,11 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm {
 				EncryptionConfig: types.EncryptionConfig{
 					AutoLockManagers: c.Spec.EncryptionConfig.AutoLockManagers,
 				},
+				CAConfig: types.CAConfig{
+					// do not include the signing CA key (it should already be redacted via the swarm APIs)
+					SigningCACert: string(c.Spec.CAConfig.SigningCACert),
+					ForceRotate:   c.Spec.CAConfig.ForceRotate,
+				},
 			},
 			TLSInfo: types.TLSInfo{
 				TrustRoot: string(c.RootCA.CACert),
@@ -114,6 +119,14 @@ func MergeSwarmSpecToGRPC(s types.Spec, spec swarmapi.ClusterSpec) (swarmapi.Clu
 	if s.CAConfig.NodeCertExpiry != 0 {
 		spec.CAConfig.NodeCertExpiry = gogotypes.DurationProto(s.CAConfig.NodeCertExpiry)
 	}
+	if s.CAConfig.SigningCACert != "" {
+		spec.CAConfig.SigningCACert = []byte(s.CAConfig.SigningCACert)
+	}
+	if s.CAConfig.SigningCAKey != "" {
+		// do propagate the signing CA key here because we want to provide it TO the swarm APIs
+		spec.CAConfig.SigningCAKey = []byte(s.CAConfig.SigningCAKey)
+	}
+	spec.CAConfig.ForceRotate = s.CAConfig.ForceRotate
 
 	for _, ca := range s.CAConfig.ExternalCAs {
 		protocol, ok := swarmapi.ExternalCA_CAProtocol_value[strings.ToUpper(string(ca.Protocol))]

docs/api/version-history.md

@@ -19,11 +19,14 @@ keywords: "API, Docker, rcli, REST, documentation"
 
 * `GET /info` now returns the list of supported logging drivers, including plugins.
 * `GET /info` and `GET /swarm` now returns the cluster-wide swarm CA info if the node is in a swarm: the cluster root CA certificate, and the cluster TLS
- leaf certificate issuer's subject and public key.
+ leaf certificate issuer's subject and public key. It also displays the desired CA signing certificate, if any was provided as part of the spec.
 * `POST /build/` now (when not silent) produces an `Aux` message in the JSON output stream with payload `types.BuildResult` for each image produced. The final such message will reference the image resulting from the build.
 * `GET /nodes` and `GET /nodes/{id}` now returns additional information about swarm TLS info if the node is part of a swarm: the trusted root CA, and the
  issuer's subject and public key.
 * `GET /distribution/(name)/json` is a new endpoint that returns a JSON output stream with payload `types.DistributionInspect` for an image name. It includes a descriptor with the digest, and supported platforms retrieved from directly contacting the registry.
+* `POST /swarm/update` now accepts 3 additional parameters as part of the swarm spec's CA configuration; the desired CA certificate for
+ the swarm, the desired CA key for the swarm (if not using an external certificate), and an optional parameter to force swarm to
+ generate and rotate to a new CA certificate/key pair.
 
 ## v1.29 API changes
 

integration-cli/docker_api_swarm_test.go

@@ -14,12 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cloudflare/cfssl/csr"
 	"github.com/cloudflare/cfssl/helpers"
+	"github.com/cloudflare/cfssl/initca"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/daemon"
+	"github.com/docker/swarmkit/ca"
 	"github.com/go-check/check"
 )
 
@@ -930,3 +933,72 @@ func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) {
 	out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top")
 	c.Assert(err, checker.IsNil, check.Commentf(out))
 }
+
+func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) {
+	m := s.AddDaemon(c, true, true)
+	w := s.AddDaemon(c, true, false)
+
+	info, err := m.SwarmInfo()
+	c.Assert(err, checker.IsNil)
+
+	currentTrustRoot := info.Cluster.TLSInfo.TrustRoot
+
+	// rotate multiple times
+	for i := 0; i < 4; i++ {
+		var cert, key []byte
+		if i%2 != 0 {
+			cert, _, key, err = initca.New(&csr.CertificateRequest{
+				CN:         "newRoot",
+				KeyRequest: csr.NewBasicKeyRequest(),
+				CA:         &csr.CAConfig{Expiry: ca.RootCAExpiration},
+			})
+			c.Assert(err, checker.IsNil)
+		}
+		expectedCert := string(cert)
+		m.UpdateSwarm(c, func(s *swarm.Spec) {
+			s.CAConfig.SigningCACert = expectedCert
+			s.CAConfig.SigningCAKey = string(key)
+			s.CAConfig.ForceRotate++
+		})
+
+		// poll to make sure update succeeds
+		var clusterTLSInfo swarm.TLSInfo
+		for j := 0; j < 18; j++ {
+			info, err := m.SwarmInfo()
+			c.Assert(err, checker.IsNil)
+			c.Assert(info.Cluster.Spec.CAConfig.SigningCACert, checker.Equals, expectedCert)
+			// the desired CA key is always redacted
+			c.Assert(info.Cluster.Spec.CAConfig.SigningCAKey, checker.Equals, "")
+
+			clusterTLSInfo = info.Cluster.TLSInfo
+
+			if !info.Cluster.RootRotationInProgress {
+				break
+			}
+
+			// root rotation not done
+			time.Sleep(250 * time.Millisecond)
+		}
+		c.Assert(clusterTLSInfo.TrustRoot, checker.Not(checker.Equals), currentTrustRoot)
+		if cert != nil {
+			c.Assert(clusterTLSInfo.TrustRoot, checker.Equals, expectedCert)
+		}
+		// could take another second or two for the nodes to trust the new roots after the've all gotten
+		// new TLS certificates
+		for j := 0; j < 18; j++ {
+			mInfo := m.GetNode(c, m.NodeID).Description.TLSInfo
+			wInfo := m.GetNode(c, w.NodeID).Description.TLSInfo
+
+			if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot {
+				break
+			}
+
+			// nodes don't trust root certs yet
+			time.Sleep(250 * time.Millisecond)
+		}
+
+		c.Assert(m.GetNode(c, m.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
+		c.Assert(m.GetNode(c, w.NodeID).Description.TLSInfo, checker.DeepEquals, clusterTLSInfo)
+		currentTrustRoot = clusterTLSInfo.TrustRoot
+	}
+}

pkg/jsonmessage/jsonmessage.go

@@ -36,7 +36,8 @@ type JSONProgress struct {
 	Total      int64 `json:"total,omitempty"`
 	Start      int64 `json:"start,omitempty"`
 	// If true, don't show xB/yB
-	HideCounts bool `json:"hidecounts,omitempty"`
+	HideCounts bool   `json:"hidecounts,omitempty"`
+	Units      string `json:"units,omitempty"`
 }
 
 func (p *JSONProgress) String() string {
@@ -55,11 +56,16 @@ func (p *JSONProgress) String() string {
 	if p.Current <= 0 && p.Total <= 0 {
 		return ""
 	}
-	current := units.HumanSize(float64(p.Current))
 	if p.Total <= 0 {
-		return fmt.Sprintf("%8v", current)
+		switch p.Units {
+		case "":
+			current := units.HumanSize(float64(p.Current))
+			return fmt.Sprintf("%8v", current)
+		default:
+			return fmt.Sprintf("%d %s", p.Current, p.Units)
+		}
 	}
-	total := units.HumanSize(float64(p.Total))
+
 	percentage := int(float64(p.Current)/float64(p.Total)*100) / 2
 	if percentage > 50 {
 		percentage = 50
@@ -73,13 +79,25 @@ func (p *JSONProgress) String() string {
 		pbBox = fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces))
 	}
 
-	if !p.HideCounts {
+	switch {
+	case p.HideCounts:
+	case p.Units == "": // no units, use bytes
+		current := units.HumanSize(float64(p.Current))
+		total := units.HumanSize(float64(p.Total))
+
 		numbersBox = fmt.Sprintf("%8v/%v", current, total)
 
 		if p.Current > p.Total {
 			// remove total display if the reported current is wonky.
 			numbersBox = fmt.Sprintf("%8v", current)
 		}
+	default:
+		numbersBox = fmt.Sprintf("%d/%d %s", p.Current, p.Total, p.Units)
+
+		if p.Current > p.Total {
+			// remove total display if the reported current is wonky.
+			numbersBox = fmt.Sprintf("%d %s", p.Current, p.Units)
+		}
 	}
 
 	if p.Current > 0 && p.Start > 0 && percentage < 50 {

pkg/jsonmessage/jsonmessage_test.go

@@ -65,22 +65,50 @@ func TestProgress(t *testing.T) {
 	if jp5.String() != expected {
 		t.Fatalf("Expected %q, got %q", expected, jp5.String())
 	}
+
+	expected = "[=========================>                         ] 50/100 units"
+	if termsz != nil && termsz.Width <= 110 {
+		expected = "    50/100 units"
+	}
+	jp6 := JSONProgress{Current: 50, Total: 100, Units: "units"}
+	if jp6.String() != expected {
+		t.Fatalf("Expected %q, got %q", expected, jp6.String())
+	}
+
+	// this number can't be negative
+	expected = "[==================================================>] 50 units"
+	if termsz != nil && termsz.Width <= 110 {
+		expected = "    50 units"
+	}
+	jp7 := JSONProgress{Current: 50, Total: 40, Units: "units"}
+	if jp7.String() != expected {
+		t.Fatalf("Expected %q, got %q", expected, jp7.String())
+	}
+
+	expected = "[=========================>                         ] "
+	if termsz != nil && termsz.Width <= 110 {
+		expected = ""
+	}
+	jp8 := JSONProgress{Current: 50, Total: 100, HideCounts: true}
+	if jp8.String() != expected {
+		t.Fatalf("Expected %q, got %q", expected, jp8.String())
+	}
 }
 
 func TestJSONMessageDisplay(t *testing.T) {
 	now := time.Now()
 	messages := map[JSONMessage][]string{
 		// Empty
-		JSONMessage{}: {"\n", "\n"},
+		{}: {"\n", "\n"},
 		// Status
-		JSONMessage{
+		{
 			Status: "status",
 		}: {
 			"status\n",
 			"status\n",
 		},
 		// General
-		JSONMessage{
+		{
 			Time:   now.Unix(),
 			ID:     "ID",
 			From:   "From",
@@ -90,7 +118,7 @@ func TestJSONMessageDisplay(t *testing.T) {
 			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(now.Unix(), 0).Format(jsonlog.RFC3339NanoFixed)),
 		},
 		// General, with nano precision time
-		JSONMessage{
+		{
 			TimeNano: now.UnixNano(),
 			ID:       "ID",
 			From:     "From",
@@ -100,7 +128,7 @@ func TestJSONMessageDisplay(t *testing.T) {
 			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
 		},
 		// General, with both times Nano is preferred
-		JSONMessage{
+		{
 			Time:     now.Unix(),
 			TimeNano: now.UnixNano(),
 			ID:       "ID",
@@ -111,23 +139,23 @@ func TestJSONMessageDisplay(t *testing.T) {
 			fmt.Sprintf("%v ID: (from From) status\n", time.Unix(0, now.UnixNano()).Format(jsonlog.RFC3339NanoFixed)),
 		},
 		// Stream over status
-		JSONMessage{
+		{
 			Status: "status",
 			Stream: "stream",
 		}: {
 			"stream",
 			"stream",
 		},
 		// With progress message
-		JSONMessage{
+		{
 			Status:          "status",
 			ProgressMessage: "progressMessage",
 		}: {
 			"status progressMessage",
 			"status progressMessage",
 		},
 		// With progress, stream empty
-		JSONMessage{
+		{
 			Status:   "status",
 			Stream:   "",
 			Progress: &JSONProgress{Current: 1},

pkg/progress/progress.go

@@ -18,6 +18,8 @@ type Progress struct {
 
 	// If true, don't show xB/yB
 	HideCounts bool
+	// If not empty, use units instead of bytes for counts
+	Units string
 
 	// Aux contains extra information not presented to the user, such as
 	// digests for push signing.

pkg/streamformatter/streamformatter.go

@@ -117,7 +117,7 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error {
 	if prog.Message != "" {
 		formatted = out.sf.formatStatus(prog.ID, prog.Message)
 	} else {
-		jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total, HideCounts: prog.HideCounts}
+		jsonProgress := jsonmessage.JSONProgress{Current: prog.Current, Total: prog.Total, HideCounts: prog.HideCounts, Units: prog.Units}
 		formatted = out.sf.formatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux)
 	}
 	_, err := out.out.Write(formatted)