toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge component 'engine' from git@github.com:docker/engine master

Browse Source
This commit is contained in:
GordonTheTurtle
2018-06-08 17:05:09 +00:00
parent 2a42a9a588 6a80f2d39d
commit 3dfd20c925
29 changed files with 503 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/engine/MAINTAINERS
View File

@ -40,6 +40,7 @@
"mlaventure",
"runcom",
"stevvooe",
"thajeztah",
"tianon",
"tibor",
"tonistiigi",

4
components/engine/api/swagger.yaml
View File

@ -2721,6 +2721,10 @@ definitions:
- "default"
- "process"
- "hyperv"
Init:
description: "Run an init inside the container that forwards signals and reaps processes. This field is omitted if empty, and the default (as configured on the daemon) is used."
type: "boolean"
x-nullable: true
NetworkAttachmentSpec:
description: |
Read-only spec type for non-swarm containers attached to swarm overlay

1
components/engine/api/types/swarm/container.go
View File

@ -55,6 +55,7 @@ type ContainerSpec struct {
User string `json:",omitempty"`
Groups []string `json:",omitempty"`
Privileges *Privileges `json:",omitempty"`
Init *bool `json:",omitempty"`
StopSignal string `json:",omitempty"`
TTY bool `json:",omitempty"`
OpenStdin bool `json:",omitempty"`

8
components/engine/builder/dockerfile/dispatchers.go
View File

@ -28,7 +28,6 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// ENV foo bar
@ -305,10 +304,12 @@ func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error {
comment := "WORKDIR " + runConfig.WorkingDir
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, d.state.operatingSystem))
containerID, err := d.builder.probeAndCreate(d.state, runConfigWithCommentCmd)
if err != nil || containerID == "" {
return err
}
if err := d.builder.docker.ContainerCreateWorkdir(containerID); err != nil {
return err
}
@ -350,8 +351,7 @@ func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error {
runConfigForCacheProbe := copyRunConfig(stateRunConfig,
withCmd(saveCmd),
withEntrypointOverride(saveCmd, nil))
hit, err := d.builder.probeCache(d.state, runConfigForCacheProbe)
if err != nil || hit {
if hit, err := d.builder.probeCache(d.state, runConfigForCacheProbe); err != nil || hit {
return err
}
@ -363,11 +363,11 @@ func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error {
// set config as already being escaped, this prevents double escaping on windows
runConfig.ArgsEscaped = true
logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd)
cID, err := d.builder.create(runConfig)
if err != nil {
return err
}
if err := d.builder.containerManager.Run(d.builder.clientCtx, cID, d.builder.Stdout, d.builder.Stderr); err != nil {
if err, ok := err.(*statusCodeError); ok {
// TODO: change error type, because jsonmessage.JSONError assumes HTTP

15
components/engine/builder/dockerfile/internals.go
View File

@ -27,6 +27,7 @@ import (
"github.com/docker/docker/pkg/system"
"github.com/docker/go-connections/nat"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// Archiver defines an interface for copying files from one destination to
@ -84,12 +85,8 @@ func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
}
runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.operatingSystem))
hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd)
if err != nil || hit {
return err
}
id, err := b.create(runConfigWithCommentCmd)
if err != nil {
id, err := b.probeAndCreate(dispatchState, runConfigWithCommentCmd)
if err != nil || id == "" {
return err
}
@ -413,13 +410,11 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
return "", err
}
// Set a log config to override any default value set on the daemon
hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
container, err := b.containerManager.Create(runConfig, hostConfig)
return container.ID, err
return b.create(runConfig)
}
func (b *Builder) create(runConfig *container.Config) (string, error) {
logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd)
hostConfig := hostConfigFromOptions(b.options)
container, err := b.containerManager.Create(runConfig, hostConfig)
if err != nil {

2
components/engine/contrib/syntax/vim/ftdetect/dockerfile.vim
View File

@ -1 +1 @@
au BufNewFile,BufRead [Dd]ockerfile,Dockerfile.*,*.Dockerfile set filetype=dockerfile
au BufNewFile,BufRead [Dd]ockerfile,[Dd]ockerfile.*,*.[Dd]ockerfile set filetype=dockerfile

17
components/engine/daemon/cluster/convert/container.go
View File

@ -35,6 +35,7 @@ func containerSpecFromGRPC(c *swarmapi.ContainerSpec) *types.ContainerSpec {
Secrets: secretReferencesFromGRPC(c.Secrets),
Configs: configReferencesFromGRPC(c.Configs),
Isolation: IsolationFromGRPC(c.Isolation),
Init: initFromGRPC(c.Init),
}
if c.DNSConfig != nil {
@ -119,6 +120,21 @@ func containerSpecFromGRPC(c *swarmapi.ContainerSpec) *types.ContainerSpec {
return containerSpec
}
func initFromGRPC(v *gogotypes.BoolValue) *bool {
if v == nil {
return nil
}
value := v.GetValue()
return &value
}
func initToGRPC(v *bool) *gogotypes.BoolValue {
if v == nil {
return nil
}
return &gogotypes.BoolValue{Value: *v}
}
func secretReferencesToGRPC(sr []*types.SecretReference) []*swarmapi.SecretReference {
refs := make([]*swarmapi.SecretReference, 0, len(sr))
for _, s := range sr {
@ -234,6 +250,7 @@ func containerToGRPC(c *types.ContainerSpec) (*swarmapi.ContainerSpec, error) {
Secrets: secretReferencesToGRPC(c.Secrets),
Configs: configReferencesToGRPC(c.Configs),
Isolation: isolationToGRPC(c.Isolation),
Init: initToGRPC(c.Init),
}
if c.DNSConfig != nil {

9
components/engine/daemon/cluster/executor/container/container.go
View File

@ -172,6 +172,14 @@ func (c *containerConfig) isolation() enginecontainer.Isolation {
return convert.IsolationFromGRPC(c.spec().Isolation)
}
func (c *containerConfig) init() *bool {
if c.spec().Init == nil {
return nil
}
init := c.spec().Init.GetValue()
return &init
}
func (c *containerConfig) exposedPorts() map[nat.Port]struct{} {
exposedPorts := make(map[nat.Port]struct{})
if c.task.Endpoint == nil {
@ -355,6 +363,7 @@ func (c *containerConfig) hostConfig() *enginecontainer.HostConfig {
Mounts: c.mounts(),
ReadonlyRootfs: c.spec().ReadOnly,
Isolation: c.isolation(),
Init: c.init(),
}
if c.spec().DNSConfig != nil {

3
components/engine/daemon/graphdriver/overlay/overlay.go
View File

@ -366,6 +366,9 @@ func (d *Driver) dir(id string) string {
// Remove cleans the directories that are created for this id.
func (d *Driver) Remove(id string) error {
if id == "" {
return fmt.Errorf("refusing to remove the directories: id is empty")
}
d.locker.Lock(id)
defer d.locker.Unlock(id)
return system.EnsureRemoveAll(d.dir(id))

7
components/engine/daemon/graphdriver/overlay2/overlay.go
View File

@ -513,12 +513,17 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
// Remove cleans the directories that are created for this id.
func (d *Driver) Remove(id string) error {
if id == "" {
return fmt.Errorf("refusing to remove the directories: id is empty")
}
d.locker.Lock(id)
defer d.locker.Unlock(id)
dir := d.dir(id)
lid, err := ioutil.ReadFile(path.Join(dir, "link"))
if err == nil {
if err := os.RemoveAll(path.Join(d.home, linkDir, string(lid))); err != nil {
if len(lid) == 0 {
logrus.WithField("storage-driver", "overlay2").Errorf("refusing to remove empty link for layer %v", id)
} else if err := os.RemoveAll(path.Join(d.home, linkDir, string(lid))); err != nil {
logrus.WithField("storage-driver", "overlay2").Debugf("Failed to remove link: %v", err)
}
}

6
components/engine/daemon/images/image_pull.go
View File

@ -5,6 +5,7 @@ import (
"io"
"runtime"
"strings"
"time"
dist "github.com/docker/distribution"
"github.com/docker/distribution/reference"
@ -20,6 +21,7 @@ import (
// PullImage initiates a pull operation. image is the repository name to pull, and
// tag may be either empty, or indicate a specific tag to pull.
func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
start := time.Now()
// Special case: "pull -a" may send an image name with a
// trailing :. This is ugly, but let's not break API
// compatibility.
@ -44,7 +46,9 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, met
}
}
return i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream)
err = i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream)
imageActions.WithValues("pull").UpdateSince(start)
return err
}
func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {

3
components/engine/daemon/images/image_push.go
View File

@ -3,6 +3,7 @@ package images // import "github.com/docker/docker/daemon/images"
import (
"context"
"io"
"time"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
@ -14,6 +15,7 @@ import (
// PushImage initiates a push operation on the repository named localName.
func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
start := time.Now()
ref, err := reference.ParseNormalizedNamed(image)
if err != nil {
return err
@ -59,5 +61,6 @@ func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHea
err = distribution.Push(ctx, ref, imagePushConfig)
close(progressChan)
<-writesDone
imageActions.WithValues("push").UpdateSince(start)
return err
}

2
components/engine/docs/contributing/set-up-dev-env.md
View File

@ -93,7 +93,7 @@ can take over 15 minutes to complete.
1. Open a terminal.
For [Docker Toolbox](../../toolbox/overview.md) users, use `docker-machine status your_vm_name` to make sure your VM is running. You
For [Docker Toolbox](https://github.com/docker/toolbox) users, use `docker-machine status your_vm_name` to make sure your VM is running. You
may need to run `eval "$(docker-machine env your_vm_name)"` to initialize your
shell environment. If you use Docker for Mac or Docker for Windows, you do not need
to use Docker Machine.

10
components/engine/integration/container/links_linux_test.go
View File

@ -41,15 +41,17 @@ func TestLinksContainerNames(t *testing.T) {
client := request.NewAPIClient(t)
ctx := context.Background()
container.Run(t, ctx, client, container.WithName("first"))
container.Run(t, ctx, client, container.WithName("second"), container.WithLinks("first:first"))
containerA := "first_" + t.Name()
containerB := "second_" + t.Name()
container.Run(t, ctx, client, container.WithName(containerA))
container.Run(t, ctx, client, container.WithName(containerB), container.WithLinks(containerA+":"+containerA))
f := filters.NewArgs(filters.Arg("name", "first"))
f := filters.NewArgs(filters.Arg("name", containerA))
containers, err := client.ContainerList(ctx, types.ContainerListOptions{
Filters: f,
})
assert.NilError(t, err)
assert.Check(t, is.Equal(1, len(containers)))
assert.Check(t, is.DeepEqual([]string{"/first", "/second/first"}, containers[0].Names))
assert.Check(t, is.DeepEqual([]string{"/" + containerA, "/" + containerB + "/" + containerA}, containers[0].Names))
}

35
components/engine/integration/internal/network/network.go Normal file
View File

@ -0,0 +1,35 @@
package network
import (
"context"
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/gotestyourself/gotestyourself/assert"
)
func createNetwork(ctx context.Context, client client.APIClient, name string, ops ...func(*types.NetworkCreate)) (string, error) {
config := types.NetworkCreate{}
for _, op := range ops {
op(&config)
}
n, err := client.NetworkCreate(ctx, name, config)
return n.ID, err
}
// Create creates a network with the specified options
func Create(ctx context.Context, client client.APIClient, name string, ops ...func(*types.NetworkCreate)) (string, error) {
return createNetwork(ctx, client, name, ops...)
}
// CreateNoError creates a network with the specified options and verifies there were no errors
func CreateNoError(t *testing.T, ctx context.Context, client client.APIClient, name string, ops ...func(*types.NetworkCreate)) string { // nolint: golint
t.Helper()
name, err := createNetwork(ctx, client, name, ops...)
assert.NilError(t, err)
return name
}

57
components/engine/integration/internal/network/ops.go Normal file
View File

@ -0,0 +1,57 @@
package network
import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
)
// WithDriver sets the driver of the network
func WithDriver(driver string) func(*types.NetworkCreate) {
return func(n *types.NetworkCreate) {
n.Driver = driver
}
}
// WithIPv6 Enables IPv6 on the network
func WithIPv6() func(*types.NetworkCreate) {
return func(n *types.NetworkCreate) {
n.EnableIPv6 = true
}
}
// WithMacvlan sets the network as macvlan with the specified parent
func WithMacvlan(parent string) func(*types.NetworkCreate) {
return func(n *types.NetworkCreate) {
n.Driver = "macvlan"
if parent != "" {
n.Options = map[string]string{
"parent": parent,
}
}
}
}
// WithOption adds the specified key/value pair to network's options
func WithOption(key, value string) func(*types.NetworkCreate) {
return func(n *types.NetworkCreate) {
if n.Options == nil {
n.Options = map[string]string{}
}
n.Options[key] = value
}
}
// WithIPAM adds an IPAM with the specified Subnet and Gateway to the network
func WithIPAM(subnet, gateway string) func(*types.NetworkCreate) {
return func(n *types.NetworkCreate) {
if n.IPAM == nil {
n.IPAM = &network.IPAM{}
}
n.IPAM.Config = append(n.IPAM.Config, network.IPAMConfig{
Subnet: subnet,
Gateway: gateway,
AuxAddress: map[string]string{},
})
}
}

8
components/engine/integration/internal/swarm/service.go
View File

@ -86,6 +86,14 @@ func defaultServiceSpec() swarmtypes.ServiceSpec {
return spec
}
// ServiceWithInit sets whether the service should use init or not
func ServiceWithInit(b *bool) func(*swarmtypes.ServiceSpec) {
return func(spec *swarmtypes.ServiceSpec) {
ensureContainerSpec(spec)
spec.TaskTemplate.ContainerSpec.Init = b
}
}
// ServiceWithImage sets the image to use for the service
func ServiceWithImage(image string) func(*swarmtypes.ServiceSpec) {
return func(spec *swarmtypes.ServiceSpec) {

139
components/engine/integration/network/macvlan/macvlan_test.go
View File

@ -7,9 +7,9 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/docker/docker/integration/internal/container"
net "github.com/docker/docker/integration/internal/network"
n "github.com/docker/docker/integration/network"
"github.com/docker/docker/internal/test/daemon"
"github.com/gotestyourself/gotestyourself/assert"
@ -33,16 +33,13 @@ func TestDockerNetworkMacvlanPersistance(t *testing.T) {
client, err := d.NewClient()
assert.NilError(t, err)
_, err = client.NetworkCreate(context.Background(), "dm-persist", types.NetworkCreate{
Driver: "macvlan",
Options: map[string]string{
"parent": "dm-dummy0.60",
},
})
assert.NilError(t, err)
assert.Check(t, n.IsNetworkAvailable(client, "dm-persist"))
netName := "dm-persist"
net.CreateNoError(t, context.Background(), client, netName,
net.WithMacvlan("dm-dummy0.60"),
)
assert.Check(t, n.IsNetworkAvailable(client, netName))
d.Restart(t)
assert.Check(t, n.IsNetworkAvailable(client, "dm-persist"))
assert.Check(t, n.IsNetworkAvailable(client, netName))
}
func TestDockerNetworkMacvlan(t *testing.T) {
@ -91,29 +88,25 @@ func testMacvlanOverlapParent(client client.APIClient) func(*testing.T) {
n.CreateMasterDummy(t, master)
defer n.DeleteInterface(t, master)
_, err := client.NetworkCreate(context.Background(), "dm-subinterface", types.NetworkCreate{
Driver: "macvlan",
Options: map[string]string{
"parent": "dm-dummy0.40",
},
})
assert.NilError(t, err)
assert.Check(t, n.IsNetworkAvailable(client, "dm-subinterface"))
netName := "dm-subinterface"
parentName := "dm-dummy0.40"
net.CreateNoError(t, context.Background(), client, netName,
net.WithMacvlan(parentName),
)
assert.Check(t, n.IsNetworkAvailable(client, netName))
_, err = client.NetworkCreate(context.Background(), "dm-parent-net-overlap", types.NetworkCreate{
Driver: "macvlan",
Options: map[string]string{
"parent": "dm-dummy0.40",
},
})
_, err := net.Create(context.Background(), client, "dm-parent-net-overlap",
net.WithMacvlan(parentName),
)
assert.Check(t, err != nil)
// delete the network while preserving the parent link
err = client.NetworkRemove(context.Background(), "dm-subinterface")
err = client.NetworkRemove(context.Background(), netName)
assert.NilError(t, err)
assert.Check(t, n.IsNetworkNotAvailable(client, "dm-subinterface"))
assert.Check(t, n.IsNetworkNotAvailable(client, netName))
// verify the network delete did not delete the predefined link
n.LinkExists(t, "dm-dummy0")
n.LinkExists(t, master)
}
}
@ -121,26 +114,24 @@ func testMacvlanSubinterface(client client.APIClient) func(*testing.T) {
return func(t *testing.T) {
// verify the same parent interface cannot be used if already in use by an existing network
master := "dm-dummy0"
parentName := "dm-dummy0.20"
n.CreateMasterDummy(t, master)
defer n.DeleteInterface(t, master)
n.CreateVlanInterface(t, master, "dm-dummy0.20", "20")
n.CreateVlanInterface(t, master, parentName, "20")
_, err := client.NetworkCreate(context.Background(), "dm-subinterface", types.NetworkCreate{
Driver: "macvlan",
Options: map[string]string{
"parent": "dm-dummy0.20",
},
})
assert.NilError(t, err)
assert.Check(t, n.IsNetworkAvailable(client, "dm-subinterface"))
netName := "dm-subinterface"
net.CreateNoError(t, context.Background(), client, netName,
net.WithMacvlan(parentName),
)
assert.Check(t, n.IsNetworkAvailable(client, netName))
// delete the network while preserving the parent link
err = client.NetworkRemove(context.Background(), "dm-subinterface")
err := client.NetworkRemove(context.Background(), netName)
assert.NilError(t, err)
assert.Check(t, n.IsNetworkNotAvailable(client, "dm-subinterface"))
assert.Check(t, n.IsNetworkNotAvailable(client, netName))
// verify the network delete did not delete the predefined link
n.LinkExists(t, "dm-dummy0.20")
n.LinkExists(t, parentName)
}
}
@ -190,34 +181,17 @@ func testMacvlanInternalMode(client client.APIClient) func(*testing.T) {
func testMacvlanMultiSubnet(client client.APIClient) func(*testing.T) {
return func(t *testing.T) {
_, err := client.NetworkCreate(context.Background(), "dualstackbridge", types.NetworkCreate{
Driver: "macvlan",
EnableIPv6: true,
IPAM: &network.IPAM{
Config: []network.IPAMConfig{
{
Subnet: "172.28.100.0/24",
AuxAddress: map[string]string{},
},
{
Subnet: "172.28.102.0/24",
Gateway: "172.28.102.254",
AuxAddress: map[string]string{},
},
{
Subnet: "2001:db8:abc2::/64",
AuxAddress: map[string]string{},
},
{
Subnet: "2001:db8:abc4::/64",
Gateway: "2001:db8:abc4::254",
AuxAddress: map[string]string{},
},
},
},
})
assert.NilError(t, err)
assert.Check(t, n.IsNetworkAvailable(client, "dualstackbridge"))
netName := "dualstackbridge"
net.CreateNoError(t, context.Background(), client, netName,
net.WithMacvlan(""),
net.WithIPv6(),
net.WithIPAM("172.28.100.0/24", ""),
net.WithIPAM("172.28.102.0/24", "172.28.102.254"),
net.WithIPAM("2001:db8:abc2::/64", ""),
net.WithIPAM("2001:db8:abc4::/64", "2001:db8:abc4::254"),
)
assert.Check(t, n.IsNetworkAvailable(client, netName))
// start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.100.0/24 and 2001:db8:abc2::/64
ctx := context.Background()
@ -276,28 +250,15 @@ func testMacvlanMultiSubnet(client client.APIClient) func(*testing.T) {
func testMacvlanAddressing(client client.APIClient) func(*testing.T) {
return func(t *testing.T) {
// Ensure the default gateways, next-hops and default dev devices are properly set
_, err := client.NetworkCreate(context.Background(), "dualstackbridge", types.NetworkCreate{
Driver: "macvlan",
EnableIPv6: true,
Options: map[string]string{
"macvlan_mode": "bridge",
},
IPAM: &network.IPAM{
Config: []network.IPAMConfig{
{
Subnet: "172.28.130.0/24",
AuxAddress: map[string]string{},
},
{
Subnet: "2001:db8:abca::/64",
Gateway: "2001:db8:abca::254",
AuxAddress: map[string]string{},
},
},
},
})
assert.NilError(t, err)
assert.Check(t, n.IsNetworkAvailable(client, "dualstackbridge"))
netName := "dualstackbridge"
net.CreateNoError(t, context.Background(), client, netName,
net.WithMacvlan(""),
net.WithIPv6(),
net.WithOption("macvlan_mode", "bridge"),
net.WithIPAM("172.28.130.0/24", ""),
net.WithIPAM("2001:db8:abca::/64", "2001:db8:abca::254"),
)
assert.Check(t, n.IsNetworkAvailable(client, netName))
ctx := context.Background()
id1 := container.Run(t, ctx, client,

84
components/engine/integration/service/create_test.go
View File

@ -2,6 +2,7 @@ package service // import "github.com/docker/docker/integration/service"
import (
"context"
"fmt"
"io/ioutil"
"testing"
"time"
@ -11,11 +12,64 @@ import (
swarmtypes "github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"github.com/docker/docker/integration/internal/swarm"
"github.com/docker/docker/internal/test/daemon"
"github.com/gotestyourself/gotestyourself/assert"
is "github.com/gotestyourself/gotestyourself/assert/cmp"
"github.com/gotestyourself/gotestyourself/poll"
)
func TestServiceCreateInit(t *testing.T) {
defer setupTest(t)()
t.Run("daemonInitDisabled", testServiceCreateInit(false))
t.Run("daemonInitEnabled", testServiceCreateInit(true))
}
func testServiceCreateInit(daemonEnabled bool) func(t *testing.T) {
return func(t *testing.T) {
var ops = []func(*daemon.Daemon){}
if daemonEnabled {
ops = append(ops, daemon.WithInit)
}
d := swarm.NewSwarm(t, testEnv, ops...)
defer d.Stop(t)
client := d.NewClientT(t)
defer client.Close()
booleanTrue := true
booleanFalse := false
serviceID := swarm.CreateService(t, d)
poll.WaitOn(t, serviceRunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
i := inspectServiceContainer(t, client, serviceID)
// HostConfig.Init == nil means that it delegates to daemon configuration
assert.Check(t, i.HostConfig.Init == nil)
serviceID = swarm.CreateService(t, d, swarm.ServiceWithInit(&booleanTrue))
poll.WaitOn(t, serviceRunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
i = inspectServiceContainer(t, client, serviceID)
assert.Check(t, is.Equal(true, *i.HostConfig.Init))
serviceID = swarm.CreateService(t, d, swarm.ServiceWithInit(&booleanFalse))
poll.WaitOn(t, serviceRunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
i = inspectServiceContainer(t, client, serviceID)
assert.Check(t, is.Equal(false, *i.HostConfig.Init))
}
}
func inspectServiceContainer(t *testing.T, client client.APIClient, serviceID string) types.ContainerJSON {
t.Helper()
filter := filters.NewArgs()
filter.Add("label", fmt.Sprintf("com.docker.swarm.service.id=%s", serviceID))
containers, err := client.ContainerList(context.Background(), types.ContainerListOptions{Filters: filter})
assert.NilError(t, err)
assert.Check(t, is.Len(containers, 1))
i, err := client.ContainerInspect(context.Background(), containers[0].ID)
assert.NilError(t, err)
return i
}
func TestCreateServiceMultipleTimes(t *testing.T) {
defer setupTest(t)()
d := swarm.NewSwarm(t, testEnv)
@ -23,7 +77,7 @@ func TestCreateServiceMultipleTimes(t *testing.T) {
client := d.NewClientT(t)
defer client.Close()
overlayName := "overlay1"
overlayName := "overlay1_" + t.Name()
networkCreate := types.NetworkCreate{
CheckDuplicate: true,
Driver: "overlay",
@ -35,9 +89,10 @@ func TestCreateServiceMultipleTimes(t *testing.T) {
var instances uint64 = 4
serviceName := "TestService_" + t.Name()
serviceSpec := []swarm.ServiceSpecOpt{
swarm.ServiceWithReplicas(instances),
swarm.ServiceWithName("TestService"),
swarm.ServiceWithName(serviceName),
swarm.ServiceWithNetwork(overlayName),
}
@ -75,7 +130,7 @@ func TestCreateWithDuplicateNetworkNames(t *testing.T) {
client := d.NewClientT(t)
defer client.Close()
name := "foo"
name := "foo_" + t.Name()
networkCreate := types.NetworkCreate{
CheckDuplicate: false,
Driver: "bridge",
@ -95,9 +150,10 @@ func TestCreateWithDuplicateNetworkNames(t *testing.T) {
// Create Service with the same name
var instances uint64 = 1
serviceName := "top_" + t.Name()
serviceID := swarm.CreateService(t, d,
swarm.ServiceWithReplicas(instances),
swarm.ServiceWithName("top"),
swarm.ServiceWithName(serviceName),
swarm.ServiceWithNetwork(name),
)
@ -138,18 +194,20 @@ func TestCreateServiceSecretFileMode(t *testing.T) {
defer client.Close()
ctx := context.Background()
secretName := "TestSecret_" + t.Name()
secretResp, err := client.SecretCreate(ctx, swarmtypes.SecretSpec{
Annotations: swarmtypes.Annotations{
Name: "TestSecret",
Name: secretName,
},
Data: []byte("TESTSECRET"),
})
assert.NilError(t, err)
var instances uint64 = 1
serviceName := "TestService_" + t.Name()
serviceID := swarm.CreateService(t, d,
swarm.ServiceWithReplicas(instances),
swarm.ServiceWithName("TestService"),
swarm.ServiceWithName(serviceName),
swarm.ServiceWithCommand([]string{"/bin/sh", "-c", "ls -l /etc/secret || /bin/top"}),
swarm.ServiceWithSecret(&swarmtypes.SecretReference{
File: &swarmtypes.SecretReferenceFileTarget{
@ -159,7 +217,7 @@ func TestCreateServiceSecretFileMode(t *testing.T) {
Mode: 0777,
},
SecretID: secretResp.ID,
SecretName: "TestSecret",
SecretName: secretName,
}),
)
@ -189,7 +247,7 @@ func TestCreateServiceSecretFileMode(t *testing.T) {
poll.WaitOn(t, serviceIsRemoved(client, serviceID), swarm.ServicePoll)
poll.WaitOn(t, noTasks(client), swarm.ServicePoll)
err = client.SecretRemove(ctx, "TestSecret")
err = client.SecretRemove(ctx, secretName)
assert.NilError(t, err)
}
@ -201,17 +259,19 @@ func TestCreateServiceConfigFileMode(t *testing.T) {
defer client.Close()
ctx := context.Background()
configName := "TestConfig_" + t.Name()
configResp, err := client.ConfigCreate(ctx, swarmtypes.ConfigSpec{
Annotations: swarmtypes.Annotations{
Name: "TestConfig",
Name: configName,
},
Data: []byte("TESTCONFIG"),
})
assert.NilError(t, err)
var instances uint64 = 1
serviceName := "TestService_" + t.Name()
serviceID := swarm.CreateService(t, d,
swarm.ServiceWithName("TestService"),
swarm.ServiceWithName(serviceName),
swarm.ServiceWithCommand([]string{"/bin/sh", "-c", "ls -l /etc/config || /bin/top"}),
swarm.ServiceWithReplicas(instances),
swarm.ServiceWithConfig(&swarmtypes.ConfigReference{
@ -222,7 +282,7 @@ func TestCreateServiceConfigFileMode(t *testing.T) {
Mode: 0777,
},
ConfigID: configResp.ID,
ConfigName: "TestConfig",
ConfigName: configName,
}),
)
@ -252,7 +312,7 @@ func TestCreateServiceConfigFileMode(t *testing.T) {
poll.WaitOn(t, serviceIsRemoved(client, serviceID))
poll.WaitOn(t, noTasks(client))
err = client.ConfigRemove(ctx, "TestConfig")
err = client.ConfigRemove(ctx, configName)
assert.NilError(t, err)
}

6
components/engine/internal/test/daemon/daemon.go
View File

@ -66,6 +66,7 @@ type Daemon struct {
userlandProxy bool
execRoot string
experimental bool
init bool
dockerdBinary string
log logT
@ -229,7 +230,10 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
fmt.Sprintf("--userland-proxy=%t", d.userlandProxy),
)
if d.experimental {
args = append(args, "--experimental", "--init")
args = append(args, "--experimental")
}
if d.init {
args = append(args, "--init")
}
if !(d.UseDefaultHost || d.UseDefaultTLSHost) {
args = append(args, []string{"--host", d.Sock()}...)

6
components/engine/internal/test/daemon/ops.go
View File

@ -5,6 +5,12 @@ import "github.com/docker/docker/internal/test/environment"
// WithExperimental sets the daemon in experimental mode
func WithExperimental(d *Daemon) {
d.experimental = true
d.init = true
}
// WithInit sets the daemon init
func WithInit(d *Daemon) {
d.init = true
}
// WithDockerdBinary sets the dockerd binary to the specified one

1
components/engine/pkg/archive/archive.go
View File

@ -127,6 +127,7 @@ func IsArchivePath(path string) bool {
if err != nil {
return false
}
defer rdr.Close()
r := tar.NewReader(rdr)
_, err = r.Next()
return err == nil

4
components/engine/pkg/archive/diff.go
View File

@ -247,10 +247,12 @@ func applyLayerHandler(dest string, layer io.Reader, options *TarOptions, decomp
defer system.Umask(oldmask) // ignore err, ErrNotSupportedPlatform
if decompress {
layer, err = DecompressStream(layer)
decompLayer, err := DecompressStream(layer)
if err != nil {
return 0, err
}
defer decompLayer.Close()
layer = decompLayer
}
return UnpackLayer(dest, layer, options)
}

1
components/engine/plugin/blobstore.go
View File

@ -145,6 +145,7 @@ func (dm *downloadManager) Download(ctx context.Context, initialRootFS image.Roo
if err != nil {
return initialRootFS, nil, err
}
defer inflatedLayerData.Close()
digester := digest.Canonical.Digester()
if _, err := chrootarchive.ApplyLayer(dm.tmpDir, io.TeeReader(inflatedLayerData, digester.Hash())); err != nil {
return initialRootFS, nil, err

45
components/engine/plugin/executor/containerd/containerd.go
View File

@ -58,6 +58,19 @@ type Executor struct {
exitHandler ExitHandler
}
// deleteTaskAndContainer deletes plugin task and then plugin container from containerd
func deleteTaskAndContainer(ctx context.Context, cli Client, id string) {
_, _, err := cli.DeleteTask(ctx, id)
if err != nil && !errdefs.IsNotFound(err) {
logrus.WithError(err).WithField("id", id).Error("failed to delete plugin task from containerd")
}
err = cli.Delete(ctx, id)
if err != nil && !errdefs.IsNotFound(err) {
logrus.WithError(err).WithField("id", id).Error("failed to delete plugin container from containerd")
}
}
// Create creates a new container
func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error {
opts := runctypes.RuncOptions{
@ -87,34 +100,21 @@ func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteClo
_, err = e.client.Start(ctx, id, "", false, attachStreamsFunc(stdout, stderr))
if err != nil {
if _, _, err2 := e.client.DeleteTask(ctx, id); err2 != nil && !errdefs.IsNotFound(err2) {
logrus.WithError(err2).WithField("id", id).Warn("Received an error while attempting to clean up containerd plugin task after failed start")
}
if err2 := e.client.Delete(ctx, id); err2 != nil && !errdefs.IsNotFound(err2) {
logrus.WithError(err2).WithField("id", id).Warn("Received an error while attempting to clean up containerd plugin container after failed start")
}
deleteTaskAndContainer(ctx, e.client, id)
}
return err
}
// Restore restores a container
func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) error {
func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) (bool, error) {
alive, _, err := e.client.Restore(context.Background(), id, attachStreamsFunc(stdout, stderr))
if err != nil && !errdefs.IsNotFound(err) {
return err
return false, err
}
if !alive {
_, _, err = e.client.DeleteTask(context.Background(), id)
if err != nil && !errdefs.IsNotFound(err) {
logrus.WithError(err).Errorf("failed to delete container plugin %s task from containerd", id)
}
err = e.client.Delete(context.Background(), id)
if err != nil && !errdefs.IsNotFound(err) {
logrus.WithError(err).Errorf("failed to delete container plugin %s from containerd", id)
}
deleteTaskAndContainer(context.Background(), e.client, id)
}
return nil
return alive, nil
}
// IsRunning returns if the container with the given id is running
@ -133,14 +133,7 @@ func (e *Executor) Signal(id string, signal int) error {
func (e *Executor) ProcessEvent(id string, et libcontainerd.EventType, ei libcontainerd.EventInfo) error {
switch et {
case libcontainerd.EventExit:
// delete task and container
if _, _, err := e.client.DeleteTask(context.Background(), id); err != nil {
logrus.WithError(err).Errorf("failed to delete container plugin %s task from containerd", id)
}
if err := e.client.Delete(context.Background(), id); err != nil {
logrus.WithError(err).Errorf("failed to delete container plugin %s from containerd", id)
}
deleteTaskAndContainer(context.Background(), e.client, id)
return e.exitHandler.HandleExitEvent(ei.ContainerID)
}
return nil

22
components/engine/plugin/manager.go
View File

@ -37,14 +37,14 @@ var validFullID = regexp.MustCompile(`^([a-f0-9]{64})$`)
// Executor is the interface that the plugin manager uses to interact with for starting/stopping plugins
type Executor interface {
Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error
Restore(id string, stdout, stderr io.WriteCloser) error
IsRunning(id string) (bool, error)
Restore(id string, stdout, stderr io.WriteCloser) (alive bool, err error)
Signal(id string, signal int) error
}
func (pm *Manager) restorePlugin(p *v2.Plugin) error {
func (pm *Manager) restorePlugin(p *v2.Plugin, c *controller) error {
if p.IsEnabled() {
return pm.restore(p)
return pm.restore(p, c)
}
return nil
}
@ -143,12 +143,15 @@ func (pm *Manager) HandleExitEvent(id string) error {
return err
}
os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))
if err := os.RemoveAll(filepath.Join(pm.config.ExecRoot, id)); err != nil && !os.IsNotExist(err) {
logrus.WithError(err).WithField("id", id).Error("Could not remove plugin bundle dir")
}
pm.mu.RLock()
c := pm.cMap[p]
if c.exitChan != nil {
close(c.exitChan)
c.exitChan = nil // ignore duplicate events (containerd issue #2299)
}
restart := c.restart
pm.mu.RUnlock()
@ -205,12 +208,15 @@ func (pm *Manager) reload() error { // todo: restore
var wg sync.WaitGroup
wg.Add(len(plugins))
for _, p := range plugins {
c := &controller{} // todo: remove this
c := &controller{exitChan: make(chan bool)}
pm.mu.Lock()
pm.cMap[p] = c
pm.mu.Unlock()
go func(p *v2.Plugin) {
defer wg.Done()
if err := pm.restorePlugin(p); err != nil {
logrus.Errorf("failed to restore plugin '%s': %s", p.Name(), err)
if err := pm.restorePlugin(p, c); err != nil {
logrus.WithError(err).WithField("id", p.GetID()).Error("Failed to restore plugin")
return
}
@ -248,7 +254,7 @@ func (pm *Manager) reload() error { // todo: restore
if requiresManualRestore {
// if liveRestore is not enabled, the plugin will be stopped now so we should enable it
if err := pm.enable(p, c, true); err != nil {
logrus.Errorf("failed to enable plugin '%s': %s", p.Name(), err)
logrus.WithError(err).WithField("id", p.GetID()).Error("failed to enable plugin")
}
}
}(p)

29
components/engine/plugin/manager_linux.go
View File

@ -79,7 +79,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
client, err := plugins.NewClientWithTimeout(addr.Network()+"://"+addr.String(), nil, p.Timeout())
if err != nil {
c.restart = false
shutdownPlugin(p, c, pm.executor)
shutdownPlugin(p, c.exitChan, pm.executor)
return errors.WithStack(err)
}
@ -106,7 +106,7 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
c.restart = false
// While restoring plugins, we need to explicitly set the state to disabled
pm.config.Store.SetState(p, false)
shutdownPlugin(p, c, pm.executor)
shutdownPlugin(p, c.exitChan, pm.executor)
return err
}
@ -117,16 +117,15 @@ func (pm *Manager) pluginPostStart(p *v2.Plugin, c *controller) error {
return pm.save(p)
}
func (pm *Manager) restore(p *v2.Plugin) error {
func (pm *Manager) restore(p *v2.Plugin, c *controller) error {
stdout, stderr := makeLoggerStreams(p.GetID())
if err := pm.executor.Restore(p.GetID(), stdout, stderr); err != nil {
alive, err := pm.executor.Restore(p.GetID(), stdout, stderr)
if err != nil {
return err
}
if pm.config.LiveRestoreEnabled {
c := &controller{}
if isRunning, _ := pm.executor.IsRunning(p.GetID()); !isRunning {
// plugin is not running, so follow normal startup procedure
if !alive {
return pm.enable(p, c, true)
}
@ -138,10 +137,16 @@ func (pm *Manager) restore(p *v2.Plugin) error {
return pm.pluginPostStart(p, c)
}
if alive {
// TODO(@cpuguy83): Should we always just re-attach to the running plugin instead of doing this?
c.restart = false
shutdownPlugin(p, c.exitChan, pm.executor)
}
return nil
}
func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) {
func shutdownPlugin(p *v2.Plugin, ec chan bool, executor Executor) {
pluginID := p.GetID()
err := executor.Signal(pluginID, int(unix.SIGTERM))
@ -149,7 +154,7 @@ func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) {
logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
} else {
select {
case <-c.exitChan:
case <-ec:
logrus.Debug("Clean shutdown of plugin")
case <-time.After(time.Second * 10):
logrus.Debug("Force shutdown plugin")
@ -157,7 +162,7 @@ func shutdownPlugin(p *v2.Plugin, c *controller, executor Executor) {
logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err)
}
select {
case <-c.exitChan:
case <-ec:
logrus.Debug("SIGKILL plugin shutdown")
case <-time.After(time.Second * 10):
logrus.Debug("Force shutdown plugin FAILED")
@ -172,7 +177,7 @@ func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
}
c.restart = false
shutdownPlugin(p, c, pm.executor)
shutdownPlugin(p, c.exitChan, pm.executor)
pm.config.Store.SetState(p, false)
return pm.save(p)
}
@ -191,7 +196,7 @@ func (pm *Manager) Shutdown() {
}
if pm.executor != nil && p.IsEnabled() {
c.restart = false
shutdownPlugin(p, c, pm.executor)
shutdownPlugin(p, c.exitChan, pm.executor)
}
}
if err := mount.RecursiveUnmount(pm.config.Root); err != nil {

156
components/engine/plugin/manager_linux_test.go
View File

@ -3,12 +3,14 @@ package plugin // import "github.com/docker/docker/plugin"
import (
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/mount"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/plugin/v2"
"github.com/gotestyourself/gotestyourself/skip"
@ -59,7 +61,7 @@ func TestManagerWithPluginMounts(t *testing.T) {
t.Fatal(err)
}
if err := m.Remove(p1.Name(), &types.PluginRmConfig{ForceRemove: true}); err != nil {
if err := m.Remove(p1.GetID(), &types.PluginRmConfig{ForceRemove: true}); err != nil {
t.Fatal(err)
}
if mounted, err := mount.Mounted(p2Mount); !mounted || err != nil {
@ -68,17 +70,18 @@ func TestManagerWithPluginMounts(t *testing.T) {
}
func newTestPlugin(t *testing.T, name, cap, root string) *v2.Plugin {
rootfs := filepath.Join(root, name)
id := stringid.GenerateNonCryptoID()
rootfs := filepath.Join(root, id)
if err := os.MkdirAll(rootfs, 0755); err != nil {
t.Fatal(err)
}
p := v2.Plugin{PluginObj: types.Plugin{Name: name}}
p := v2.Plugin{PluginObj: types.Plugin{ID: id, Name: name}}
p.Rootfs = rootfs
iType := types.PluginInterfaceType{Capability: cap, Prefix: "docker", Version: "1.0"}
i := types.PluginConfigInterface{Socket: "plugins.sock", Types: []types.PluginInterfaceType{iType}}
i := types.PluginConfigInterface{Socket: "plugin.sock", Types: []types.PluginInterfaceType{iType}}
p.PluginObj.Config.Interface = i
p.PluginObj.ID = name
p.PluginObj.ID = id
return &p
}
@ -90,8 +93,8 @@ func (e *simpleExecutor) Create(id string, spec specs.Spec, stdout, stderr io.Wr
return errors.New("Create failed")
}
func (e *simpleExecutor) Restore(id string, stdout, stderr io.WriteCloser) error {
return nil
func (e *simpleExecutor) Restore(id string, stdout, stderr io.WriteCloser) (bool, error) {
return false, nil
}
func (e *simpleExecutor) IsRunning(id string) (bool, error) {
@ -133,7 +136,144 @@ func TestCreateFailed(t *testing.T) {
t.Fatalf("expected Create failed error, got %v", err)
}
if err := m.Remove(p.Name(), &types.PluginRmConfig{ForceRemove: true}); err != nil {
if err := m.Remove(p.GetID(), &types.PluginRmConfig{ForceRemove: true}); err != nil {
t.Fatal(err)
}
}
type executorWithRunning struct {
m *Manager
root string
exitChans map[string]chan struct{}
}
func (e *executorWithRunning) Create(id string, spec specs.Spec, stdout, stderr io.WriteCloser) error {
sockAddr := filepath.Join(e.root, id, "plugin.sock")
ch := make(chan struct{})
if e.exitChans == nil {
e.exitChans = make(map[string]chan struct{})
}
e.exitChans[id] = ch
listenTestPlugin(sockAddr, ch)
return nil
}
func (e *executorWithRunning) IsRunning(id string) (bool, error) {
return true, nil
}
func (e *executorWithRunning) Restore(id string, stdout, stderr io.WriteCloser) (bool, error) {
return true, nil
}
func (e *executorWithRunning) Signal(id string, signal int) error {
ch := e.exitChans[id]
ch <- struct{}{}
<-ch
e.m.HandleExitEvent(id)
return nil
}
func TestPluginAlreadyRunningOnStartup(t *testing.T) {
t.Parallel()
root, err := ioutil.TempDir("", t.Name())
if err != nil {
t.Fatal(err)
}
defer system.EnsureRemoveAll(root)
for _, test := range []struct {
desc string
config ManagerConfig
}{
{
desc: "live-restore-disabled",
config: ManagerConfig{
LogPluginEvent: func(_, _, _ string) {},
},
},
{
desc: "live-restore-enabled",
config: ManagerConfig{
LogPluginEvent: func(_, _, _ string) {},
LiveRestoreEnabled: true,
},
},
} {
t.Run(test.desc, func(t *testing.T) {
config := test.config
desc := test.desc
t.Parallel()
p := newTestPlugin(t, desc, desc, config.Root)
p.PluginObj.Enabled = true
// Need a short-ish path here so we don't run into unix socket path length issues.
config.ExecRoot, err = ioutil.TempDir("", "plugintest")
executor := &executorWithRunning{root: config.ExecRoot}
config.CreateExecutor = func(m *Manager) (Executor, error) { executor.m = m; return executor, nil }
if err := executor.Create(p.GetID(), specs.Spec{}, nil, nil); err != nil {
t.Fatal(err)
}
root := filepath.Join(root, desc)
config.Root = filepath.Join(root, "manager")
if err := os.MkdirAll(filepath.Join(config.Root, p.GetID()), 0755); err != nil {
t.Fatal(err)
}
if !p.IsEnabled() {
t.Fatal("plugin should be enabled")
}
if err := (&Manager{config: config}).save(p); err != nil {
t.Fatal(err)
}
s := NewStore()
config.Store = s
if err != nil {
t.Fatal(err)
}
defer system.EnsureRemoveAll(config.ExecRoot)
m, err := NewManager(config)
if err != nil {
t.Fatal(err)
}
defer m.Shutdown()
p = s.GetAll()[p.GetID()] // refresh `p` with what the manager knows
if p.Client() == nil {
t.Fatal("plugin client should not be nil")
}
})
}
}
func listenTestPlugin(sockAddr string, exit chan struct{}) (net.Listener, error) {
if err := os.MkdirAll(filepath.Dir(sockAddr), 0755); err != nil {
return nil, err
}
l, err := net.Listen("unix", sockAddr)
if err != nil {
return nil, err
}
go func() {
for {
conn, err := l.Accept()
if err != nil {
return
}
conn.Close()
}
}()
go func() {
<-exit
l.Close()
os.Remove(sockAddr)
exit <- struct{}{}
}()
return l, nil
}

2
components/engine/plugin/manager_windows.go
View File

@ -19,7 +19,7 @@ func (pm *Manager) disable(p *v2.Plugin, c *controller) error {
return fmt.Errorf("Not implemented")
}
func (pm *Manager) restore(p *v2.Plugin) error {
func (pm *Manager) restore(p *v2.Plugin, c *controller) error {
return fmt.Errorf("Not implemented")
}