Remove CopyOnBuild from the daemon.

Add CreateImage() to the daemon
Refactor daemon.Comit() and expose a Image.NewChild()
Update copy to use IDMappings.

Signed-off-by: Daniel Nephin <[email protected]>

Author: dnephin

builder/builder.go

@@ -11,6 +11,9 @@ import (
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	containerpkg "github.com/docker/docker/container"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/idtools"
 	"golang.org/x/net/context"
 )
 
@@ -42,11 +45,9 @@ type Backend interface {
 	// ContainerCreateWorkdir creates the workdir
 	ContainerCreateWorkdir(containerID string) error
 
-	// ContainerCopy copies/extracts a source FileInfo to a destination path inside a container
-	// specified by a container object.
-	// TODO: extract in the builder instead of passing `decompress`
-	// TODO: use containerd/fs.changestream instead as a source
-	CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
+	CreateImage(config []byte, parent string) (string, error)
+
+	IDMappings() *idtools.IDMappings
 
 	ImageCacheBuilder
 }
@@ -96,10 +97,13 @@ type ImageCache interface {
 type Image interface {
 	ImageID() string
 	RunConfig() *container.Config
+	MarshalJSON() ([]byte, error)
+	NewChild(child image.ChildConfig) *image.Image
 }
 
 // ReleaseableLayer is an image layer that can be mounted and released
 type ReleaseableLayer interface {
 	Release() error
 	Mount() (string, error)
+	DiffID() layer.DiffID
 }

builder/dockerfile/builder.go

@@ -15,6 +15,8 @@ import (
 	"github.com/docker/docker/builder/dockerfile/command"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
@@ -98,6 +100,7 @@ type Builder struct {
 	docker    builder.Backend
 	clientCtx context.Context
 
+	archiver         *archive.Archiver
 	buildStages      *buildStages
 	disableCommit    bool
 	buildArgs        *buildArgs
@@ -121,6 +124,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		Aux:              options.ProgressWriter.AuxFormatter,
 		Output:           options.ProgressWriter.Output,
 		docker:           options.Backend,
+		archiver:         chrootarchive.NewArchiver(options.Backend.IDMappings()),
 		buildArgs:        newBuildArgs(config.BuildArgs),
 		buildStages:      newBuildStages(),
 		imageSources:     newImageSources(clientCtx, options),

builder/dockerfile/copy.go

@@ -13,9 +13,12 @@ import (
 
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/pkg/errors"
@@ -34,6 +37,10 @@ type copyInfo struct {
 	hash string
 }
 
+func (c copyInfo) fullPath() (string, error) {
+	return symlink.FollowSymlinkInScope(filepath.Join(c.root, c.path), c.root)
+}
+
 func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
 	return copyInfo{root: source.Root(), path: path, hash: hash}
 }
@@ -355,3 +362,53 @@ func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote b
 	lc, err := remotecontext.NewLazyContext(tmpDir)
 	return lc, filename, err
 }
+
+type copyFileOptions struct {
+	decompress bool
+	archiver   *archive.Archiver
+}
+
+func copyFile(dest copyInfo, source copyInfo, options copyFileOptions) error {
+	srcPath, err := source.fullPath()
+	if err != nil {
+		return err
+	}
+	destPath, err := dest.fullPath()
+	if err != nil {
+		return err
+	}
+
+	archiver := options.archiver
+	rootIDs := archiver.IDMappings.RootPair()
+
+	src, err := os.Stat(srcPath)
+	if err != nil {
+		return err // TODO: errors.Wrapf
+	}
+	if src.IsDir() {
+		if err := archiver.CopyWithTar(srcPath, destPath); err != nil {
+			return err
+		}
+		return fixPermissions(srcPath, destPath, rootIDs)
+	}
+
+	if options.decompress && archive.IsArchivePath(srcPath) {
+		// To support the untar feature we need to clean up the path a little bit
+		// because tar is not very forgiving
+		tarDest := dest.path
+		// TODO: could this be just TrimSuffix()?
+		if strings.HasSuffix(tarDest, string(os.PathSeparator)) {
+			tarDest = filepath.Dir(dest.path)
+		}
+		return archiver.UntarPath(srcPath, tarDest)
+	}
+
+	if err := idtools.MkdirAllAndChownNew(filepath.Dir(destPath), 0755, rootIDs); err != nil {
+		return err
+	}
+	if err := archiver.CopyFileWithTar(srcPath, destPath); err != nil {
+		return err
+	}
+	// TODO: do I have to change destPath to the filename?
+	return fixPermissions(srcPath, destPath, rootIDs)
+}

builder/dockerfile/copy_unix.go

@@ -0,0 +1,64 @@
+package dockerfile
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/idtools"
+)
+
+func pathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	switch {
+	case err == nil:
+		return true, nil
+	case os.IsNotExist(err):
+		return false, nil
+	}
+	return false, err
+}
+
+// TODO: review this
+func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
+	doChownDestination, err := chownDestinationRoot(destination)
+	if err != nil {
+		return err
+	}
+
+	// We Walk on the source rather than on the destination because we don't
+	// want to change permissions on things we haven't created or modified.
+	return filepath.Walk(source, func(fullpath string, info os.FileInfo, err error) error {
+		// Do not alter the walk root iff. it existed before, as it doesn't fall under
+		// the domain of "things we should chown".
+		if !doChownDestination && (source == fullpath) {
+			return nil
+		}
+
+		// Path is prefixed by source: substitute with destination instead.
+		cleaned, err := filepath.Rel(source, fullpath)
+		if err != nil {
+			return err
+		}
+
+		fullpath = filepath.Join(destination, cleaned)
+		return os.Lchown(fullpath, rootIDs.UID, rootIDs.GID)
+	})
+}
+
+// If the destination didn't already exist, or the destination isn't a
+// directory, then we should Lchown the destination. Otherwise, we shouldn't
+// Lchown the destination.
+func chownDestinationRoot(destination string) (bool, error) {
+	destExists, err := pathExists(destination)
+	if err != nil {
+		return false, err
+	}
+	destStat, err := os.Stat(destination)
+	if err != nil {
+		// This should *never* be reached, because the destination must've already
+		// been created while untar-ing the context.
+		return false, err
+	}
+
+	return !destExists || !destStat.IsDir(), nil
+}

builder/dockerfile/copy_windows.go

@@ -0,0 +1,8 @@
+package dockerfile
+
+import "github.com/docker/docker/pkg/idtools"
+
+func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
+	// chown is not supported on Windows
+	return nil
+}

builder/dockerfile/dispatchers.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/api/types/strslice"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/parser"
+	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/signal"
 	"github.com/docker/go-connections/nat"
@@ -251,10 +252,8 @@ func parseBuildStageName(args []string) (string, error) {
 	return stageName, nil
 }
 
-// scratchImage is used as a token for the empty base image. It uses buildStage
-// as a convenient implementation of builder.Image, but is not actually a
-// buildStage.
-var scratchImage builder.Image = &buildStage{}
+// scratchImage is used as a token for the empty base image.
+var scratchImage builder.Image = &image.Image{}
 
 func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, error) {
 	substitutionArgs := []string{}
@@ -267,8 +266,8 @@ func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, err
 		return nil, err
 	}
 
-	if im, ok := b.buildStages.getByName(name); ok {
-		return im, nil
+	if stage, ok := b.buildStages.getByName(name); ok {
+		name = stage.ImageID()
 	}
 
 	// Windows cannot support a container with no base image.

builder/dockerfile/imagecontext.go

@@ -6,37 +6,29 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types/backend"
-	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/layer"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
 type buildStage struct {
-	id     string
-	config *container.Config
+	id string
 }
 
-func newBuildStageFromImage(image builder.Image) *buildStage {
-	return &buildStage{id: image.ImageID(), config: image.RunConfig()}
+func newBuildStage(imageID string) *buildStage {
+	return &buildStage{id: imageID}
 }
 
 func (b *buildStage) ImageID() string {
 	return b.id
 }
 
-func (b *buildStage) RunConfig() *container.Config {
-	return b.config
-}
-
-func (b *buildStage) update(imageID string, runConfig *container.Config) {
+func (b *buildStage) update(imageID string) {
 	b.id = imageID
-	b.config = runConfig
 }
 
-var _ builder.Image = &buildStage{}
-
 // buildStages tracks each stage of a build so they can be retrieved by index
 // or by name.
 type buildStages struct {
@@ -48,12 +40,12 @@ func newBuildStages() *buildStages {
 	return &buildStages{byName: make(map[string]*buildStage)}
 }
 
-func (s *buildStages) getByName(name string) (builder.Image, bool) {
+func (s *buildStages) getByName(name string) (*buildStage, bool) {
 	stage, ok := s.byName[strings.ToLower(name)]
 	return stage, ok
 }
 
-func (s *buildStages) get(indexOrName string) (builder.Image, error) {
+func (s *buildStages) get(indexOrName string) (*buildStage, error) {
 	index, err := strconv.Atoi(indexOrName)
 	if err == nil {
 		if err := s.validateIndex(index); err != nil {
@@ -78,7 +70,7 @@ func (s *buildStages) validateIndex(i int) error {
 }
 
 func (s *buildStages) add(name string, image builder.Image) error {
-	stage := newBuildStageFromImage(image)
+	stage := newBuildStage(image.ImageID())
 	name = strings.ToLower(name)
 	if len(name) > 0 {
 		if _, ok := s.byName[name]; ok {
@@ -90,8 +82,8 @@ func (s *buildStages) add(name string, image builder.Image) error {
 	return nil
 }
 
-func (s *buildStages) update(imageID string, runConfig *container.Config) {
-	s.sequence[len(s.sequence)-1].update(imageID, runConfig)
+func (s *buildStages) update(imageID string) {
+	s.sequence[len(s.sequence)-1].update(imageID)
 }
 
 type getAndMountFunc func(string) (builder.Image, builder.ReleaseableLayer, error)
@@ -190,3 +182,7 @@ func (im *imageMount) Image() builder.Image {
 func (im *imageMount) ImageID() string {
 	return im.image.ImageID()
 }
+
+func (im *imageMount) DiffID() layer.DiffID {
+	return im.layer.DiffID()
+}