Merge pull request #29003 from dnephin/pkg-compose-transform

Move composefile -> engine api type conversion to `cli/compose/convert`
This commit is contained in:
Vincent Demeester
2016-12-16 23:34:41 +01:00
committed by GitHub
12 changed files with 962 additions and 525 deletions
+15 -21
View File
@@ -6,24 +6,26 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/cli/compose/convert"
"github.com/docker/docker/client"
"github.com/docker/docker/opts"
)
const (
labelNamespace = "com.docker.stack.namespace"
)
func getStackLabels(namespace string, labels map[string]string) map[string]string {
if labels == nil {
labels = make(map[string]string)
}
labels[labelNamespace] = namespace
return labels
}
func getStackFilter(namespace string) filters.Args {
filter := filters.NewArgs()
filter.Add("label", labelNamespace+"="+namespace)
filter.Add("label", convert.LabelNamespace+"="+namespace)
return filter
}
func getStackFilterFromOpt(namespace string, opt opts.FilterOpt) filters.Args {
filter := opt.Value()
filter.Add("label", convert.LabelNamespace+"="+namespace)
return filter
}
func getAllStacksFilter() filters.Args {
filter := filters.NewArgs()
filter.Add("label", convert.LabelNamespace)
return filter
}
@@ -46,11 +48,3 @@ func getStackNetworks(
ctx,
types.NetworkListOptions{Filters: getStackFilter(namespace)})
}
type namespace struct {
name string
}
func (n namespace) scope(name string) string {
return n.name + "_" + name
}
+10 -485
View File
@@ -7,7 +7,6 @@ import (
"os"
"sort"
"strings"
"time"
"github.com/spf13/cobra"
"golang.org/x/net/context"
@@ -15,16 +14,11 @@ import (
"github.com/aanand/compose-file/loader"
composetypes "github.com/aanand/compose-file/types"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount"
networktypes "github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/cli"
"github.com/docker/docker/cli/command"
"github.com/docker/docker/cli/compose/convert"
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/opts"
runconfigopts "github.com/docker/docker/runconfig/opts"
"github.com/docker/go-connections/nat"
)
const (
@@ -121,16 +115,16 @@ func deployCompose(ctx context.Context, dockerCli *command.DockerCli, opts deplo
return err
}
namespace := namespace{name: opts.namespace}
namespace := convert.NewNamespace(opts.namespace)
networks, externalNetworks := convertNetworks(namespace, config.Networks)
networks, externalNetworks := convert.Networks(namespace, config.Networks)
if err := validateExternalNetworks(ctx, dockerCli, externalNetworks); err != nil {
return err
}
if err := createNetworks(ctx, dockerCli, namespace, networks); err != nil {
return err
}
services, err := convertServices(namespace, config)
services, err := convert.Services(namespace, config)
if err != nil {
return err
}
@@ -179,51 +173,6 @@ func getConfigFile(filename string) (*composetypes.ConfigFile, error) {
}, nil
}
func convertNetworks(
namespace namespace,
networks map[string]composetypes.NetworkConfig,
) (map[string]types.NetworkCreate, []string) {
if networks == nil {
networks = make(map[string]composetypes.NetworkConfig)
}
// TODO: only add default network if it's used
networks["default"] = composetypes.NetworkConfig{}
externalNetworks := []string{}
result := make(map[string]types.NetworkCreate)
for internalName, network := range networks {
if network.External.External {
externalNetworks = append(externalNetworks, network.External.Name)
continue
}
createOpts := types.NetworkCreate{
Labels: getStackLabels(namespace.name, network.Labels),
Driver: network.Driver,
Options: network.DriverOpts,
}
if network.Ipam.Driver != "" || len(network.Ipam.Config) > 0 {
createOpts.IPAM = &networktypes.IPAM{}
}
if network.Ipam.Driver != "" {
createOpts.IPAM.Driver = network.Ipam.Driver
}
for _, ipamConfig := range network.Ipam.Config {
config := networktypes.IPAMConfig{
Subnet: ipamConfig.Subnet,
}
createOpts.IPAM.Config = append(createOpts.IPAM.Config, config)
}
result[internalName] = createOpts
}
return result, externalNetworks
}
func validateExternalNetworks(
ctx context.Context,
dockerCli *command.DockerCli,
@@ -249,12 +198,12 @@ func validateExternalNetworks(
func createNetworks(
ctx context.Context,
dockerCli *command.DockerCli,
namespace namespace,
namespace convert.Namespace,
networks map[string]types.NetworkCreate,
) error {
client := dockerCli.Client()
existingNetworks, err := getStackNetworks(ctx, client, namespace.name)
existingNetworks, err := getStackNetworks(ctx, client, namespace.Name())
if err != nil {
return err
}
@@ -265,7 +214,7 @@ func createNetworks(
}
for internalName, createOpts := range networks {
name := namespace.scope(internalName)
name := namespace.Scope(internalName)
if _, exists := existingNetworkMap[name]; exists {
continue
}
@@ -283,164 +232,17 @@ func createNetworks(
return nil
}
func convertServiceNetworks(
networks map[string]*composetypes.ServiceNetworkConfig,
networkConfigs map[string]composetypes.NetworkConfig,
namespace namespace,
name string,
) ([]swarm.NetworkAttachmentConfig, error) {
if len(networks) == 0 {
return []swarm.NetworkAttachmentConfig{
{
Target: namespace.scope("default"),
Aliases: []string{name},
},
}, nil
}
nets := []swarm.NetworkAttachmentConfig{}
for networkName, network := range networks {
networkConfig, ok := networkConfigs[networkName]
if !ok {
return []swarm.NetworkAttachmentConfig{}, fmt.Errorf("invalid network: %s", networkName)
}
var aliases []string
if network != nil {
aliases = network.Aliases
}
target := namespace.scope(networkName)
if networkConfig.External.External {
target = networkName
}
nets = append(nets, swarm.NetworkAttachmentConfig{
Target: target,
Aliases: append(aliases, name),
})
}
return nets, nil
}
func convertVolumes(
serviceVolumes []string,
stackVolumes map[string]composetypes.VolumeConfig,
namespace namespace,
) ([]mount.Mount, error) {
var mounts []mount.Mount
for _, volumeSpec := range serviceVolumes {
mount, err := convertVolumeToMount(volumeSpec, stackVolumes, namespace)
if err != nil {
return nil, err
}
mounts = append(mounts, mount)
}
return mounts, nil
}
func convertVolumeToMount(
volumeSpec string,
stackVolumes map[string]composetypes.VolumeConfig,
namespace namespace,
) (mount.Mount, error) {
var source, target string
var mode []string
// TODO: split Windows path mappings properly
parts := strings.SplitN(volumeSpec, ":", 3)
switch len(parts) {
case 3:
source = parts[0]
target = parts[1]
mode = strings.Split(parts[2], ",")
case 2:
source = parts[0]
target = parts[1]
case 1:
target = parts[0]
default:
return mount.Mount{}, fmt.Errorf("invald volume: %s", volumeSpec)
}
// TODO: catch Windows paths here
if strings.HasPrefix(source, "/") {
return mount.Mount{
Type: mount.TypeBind,
Source: source,
Target: target,
ReadOnly: isReadOnly(mode),
BindOptions: getBindOptions(mode),
}, nil
}
stackVolume, exists := stackVolumes[source]
if !exists {
return mount.Mount{}, fmt.Errorf("undefined volume: %s", source)
}
var volumeOptions *mount.VolumeOptions
if stackVolume.External.Name != "" {
source = stackVolume.External.Name
} else {
volumeOptions = &mount.VolumeOptions{
Labels: getStackLabels(namespace.name, stackVolume.Labels),
NoCopy: isNoCopy(mode),
}
if stackVolume.Driver != "" {
volumeOptions.DriverConfig = &mount.Driver{
Name: stackVolume.Driver,
Options: stackVolume.DriverOpts,
}
}
source = namespace.scope(source)
}
return mount.Mount{
Type: mount.TypeVolume,
Source: source,
Target: target,
ReadOnly: isReadOnly(mode),
VolumeOptions: volumeOptions,
}, nil
}
func modeHas(mode []string, field string) bool {
for _, item := range mode {
if item == field {
return true
}
}
return false
}
func isReadOnly(mode []string) bool {
return modeHas(mode, "ro")
}
func isNoCopy(mode []string) bool {
return modeHas(mode, "nocopy")
}
func getBindOptions(mode []string) *mount.BindOptions {
for _, item := range mode {
if strings.Contains(item, "private") || strings.Contains(item, "shared") || strings.Contains(item, "slave") {
return &mount.BindOptions{Propagation: mount.Propagation(item)}
}
}
return nil
}
func deployServices(
ctx context.Context,
dockerCli *command.DockerCli,
services map[string]swarm.ServiceSpec,
namespace namespace,
namespace convert.Namespace,
sendAuth bool,
) error {
apiClient := dockerCli.Client()
out := dockerCli.Out()
existingServices, err := getServices(ctx, apiClient, namespace.name)
existingServices, err := getServices(ctx, apiClient, namespace.Name())
if err != nil {
return err
}
@@ -451,7 +253,7 @@ func deployServices(
}
for internalName, serviceSpec := range services {
name := namespace.scope(internalName)
name := namespace.Scope(internalName)
encodedAuth := ""
if sendAuth {
@@ -499,280 +301,3 @@ func deployServices(
return nil
}
func convertServices(
namespace namespace,
config *composetypes.Config,
) (map[string]swarm.ServiceSpec, error) {
result := make(map[string]swarm.ServiceSpec)
services := config.Services
volumes := config.Volumes
networks := config.Networks
for _, service := range services {
serviceSpec, err := convertService(namespace, service, networks, volumes)
if err != nil {
return nil, err
}
result[service.Name] = serviceSpec
}
return result, nil
}
func convertService(
namespace namespace,
service composetypes.ServiceConfig,
networkConfigs map[string]composetypes.NetworkConfig,
volumes map[string]composetypes.VolumeConfig,
) (swarm.ServiceSpec, error) {
name := namespace.scope(service.Name)
endpoint, err := convertEndpointSpec(service.Ports)
if err != nil {
return swarm.ServiceSpec{}, err
}
mode, err := convertDeployMode(service.Deploy.Mode, service.Deploy.Replicas)
if err != nil {
return swarm.ServiceSpec{}, err
}
mounts, err := convertVolumes(service.Volumes, volumes, namespace)
if err != nil {
// TODO: better error message (include service name)
return swarm.ServiceSpec{}, err
}
resources, err := convertResources(service.Deploy.Resources)
if err != nil {
return swarm.ServiceSpec{}, err
}
restartPolicy, err := convertRestartPolicy(
service.Restart, service.Deploy.RestartPolicy)
if err != nil {
return swarm.ServiceSpec{}, err
}
healthcheck, err := convertHealthcheck(service.HealthCheck)
if err != nil {
return swarm.ServiceSpec{}, err
}
networks, err := convertServiceNetworks(service.Networks, networkConfigs, namespace, service.Name)
if err != nil {
return swarm.ServiceSpec{}, err
}
var logDriver *swarm.Driver
if service.Logging != nil {
logDriver = &swarm.Driver{
Name: service.Logging.Driver,
Options: service.Logging.Options,
}
}
serviceSpec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: name,
Labels: getStackLabels(namespace.name, service.Deploy.Labels),
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: swarm.ContainerSpec{
Image: service.Image,
Command: service.Entrypoint,
Args: service.Command,
Hostname: service.Hostname,
Hosts: convertExtraHosts(service.ExtraHosts),
Healthcheck: healthcheck,
Env: convertEnvironment(service.Environment),
Labels: getStackLabels(namespace.name, service.Labels),
Dir: service.WorkingDir,
User: service.User,
Mounts: mounts,
StopGracePeriod: service.StopGracePeriod,
TTY: service.Tty,
OpenStdin: service.StdinOpen,
},
LogDriver: logDriver,
Resources: resources,
RestartPolicy: restartPolicy,
Placement: &swarm.Placement{
Constraints: service.Deploy.Placement.Constraints,
},
},
EndpointSpec: endpoint,
Mode: mode,
Networks: networks,
UpdateConfig: convertUpdateConfig(service.Deploy.UpdateConfig),
}
return serviceSpec, nil
}
func convertExtraHosts(extraHosts map[string]string) []string {
hosts := []string{}
for host, ip := range extraHosts {
hosts = append(hosts, fmt.Sprintf("%s %s", ip, host))
}
return hosts
}
func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container.HealthConfig, error) {
if healthcheck == nil {
return nil, nil
}
var (
err error
timeout, interval time.Duration
retries int
)
if healthcheck.Disable {
if len(healthcheck.Test) != 0 {
return nil, fmt.Errorf("command and disable key can't be set at the same time")
}
return &container.HealthConfig{
Test: []string{"NONE"},
}, nil
}
if healthcheck.Timeout != "" {
timeout, err = time.ParseDuration(healthcheck.Timeout)
if err != nil {
return nil, err
}
}
if healthcheck.Interval != "" {
interval, err = time.ParseDuration(healthcheck.Interval)
if err != nil {
return nil, err
}
}
if healthcheck.Retries != nil {
retries = int(*healthcheck.Retries)
}
return &container.HealthConfig{
Test: healthcheck.Test,
Timeout: timeout,
Interval: interval,
Retries: retries,
}, nil
}
func convertRestartPolicy(restart string, source *composetypes.RestartPolicy) (*swarm.RestartPolicy, error) {
// TODO: log if restart is being ignored
if source == nil {
policy, err := runconfigopts.ParseRestartPolicy(restart)
if err != nil {
return nil, err
}
// TODO: is this an accurate convertion?
switch {
case policy.IsNone():
return nil, nil
case policy.IsAlways(), policy.IsUnlessStopped():
return &swarm.RestartPolicy{
Condition: swarm.RestartPolicyConditionAny,
}, nil
case policy.IsOnFailure():
attempts := uint64(policy.MaximumRetryCount)
return &swarm.RestartPolicy{
Condition: swarm.RestartPolicyConditionOnFailure,
MaxAttempts: &attempts,
}, nil
}
}
return &swarm.RestartPolicy{
Condition: swarm.RestartPolicyCondition(source.Condition),
Delay: source.Delay,
MaxAttempts: source.MaxAttempts,
Window: source.Window,
}, nil
}
func convertUpdateConfig(source *composetypes.UpdateConfig) *swarm.UpdateConfig {
if source == nil {
return nil
}
parallel := uint64(1)
if source.Parallelism != nil {
parallel = *source.Parallelism
}
return &swarm.UpdateConfig{
Parallelism: parallel,
Delay: source.Delay,
FailureAction: source.FailureAction,
Monitor: source.Monitor,
MaxFailureRatio: source.MaxFailureRatio,
}
}
func convertResources(source composetypes.Resources) (*swarm.ResourceRequirements, error) {
resources := &swarm.ResourceRequirements{}
if source.Limits != nil {
cpus, err := opts.ParseCPUs(source.Limits.NanoCPUs)
if err != nil {
return nil, err
}
resources.Limits = &swarm.Resources{
NanoCPUs: cpus,
MemoryBytes: int64(source.Limits.MemoryBytes),
}
}
if source.Reservations != nil {
cpus, err := opts.ParseCPUs(source.Reservations.NanoCPUs)
if err != nil {
return nil, err
}
resources.Reservations = &swarm.Resources{
NanoCPUs: cpus,
MemoryBytes: int64(source.Reservations.MemoryBytes),
}
}
return resources, nil
}
func convertEndpointSpec(source []string) (*swarm.EndpointSpec, error) {
portConfigs := []swarm.PortConfig{}
ports, portBindings, err := nat.ParsePortSpecs(source)
if err != nil {
return nil, err
}
for port := range ports {
portConfigs = append(
portConfigs,
opts.ConvertPortToPortConfig(port, portBindings)...)
}
return &swarm.EndpointSpec{Ports: portConfigs}, nil
}
func convertEnvironment(source map[string]string) []string {
var output []string
for name, value := range source {
output = append(output, fmt.Sprintf("%s=%s", name, value))
}
return output
}
func convertDeployMode(mode string, replicas *uint64) (swarm.ServiceMode, error) {
serviceMode := swarm.ServiceMode{}
switch mode {
case "global":
if replicas != nil {
return serviceMode, fmt.Errorf("replicas can only be used with replicated mode")
}
serviceMode.Global = &swarm.GlobalService{}
case "replicated", "":
serviceMode.Replicated = &swarm.ReplicatedService{Replicas: replicas}
default:
return serviceMode, fmt.Errorf("Unknown mode: %s", mode)
}
return serviceMode, nil
}
+7 -6
View File
@@ -6,6 +6,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/cli/command"
"github.com/docker/docker/cli/compose/convert"
)
func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deployOptions) error {
@@ -18,20 +19,20 @@ func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deploy
return err
}
namespace := namespace{name: opts.namespace}
namespace := convert.NewNamespace(opts.namespace)
networks := make(map[string]types.NetworkCreate)
for _, service := range bundle.Services {
for _, networkName := range service.Networks {
networks[networkName] = types.NetworkCreate{
Labels: getStackLabels(namespace.name, nil),
Labels: convert.AddStackLabel(namespace, nil),
}
}
}
services := make(map[string]swarm.ServiceSpec)
for internalName, service := range bundle.Services {
name := namespace.scope(internalName)
name := namespace.Scope(internalName)
var ports []swarm.PortConfig
for _, portSpec := range service.Ports {
@@ -44,7 +45,7 @@ func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deploy
nets := []swarm.NetworkAttachmentConfig{}
for _, networkName := range service.Networks {
nets = append(nets, swarm.NetworkAttachmentConfig{
Target: namespace.scope(networkName),
Target: namespace.Scope(networkName),
Aliases: []string{networkName},
})
}
@@ -52,7 +53,7 @@ func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deploy
serviceSpec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: name,
Labels: getStackLabels(namespace.name, service.Labels),
Labels: convert.AddStackLabel(namespace, service.Labels),
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: swarm.ContainerSpec{
@@ -63,7 +64,7 @@ func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deploy
// Service Labels will not be copied to Containers
// automatically during the deployment so we apply
// it here.
Labels: getStackLabels(namespace.name, nil),
Labels: convert.AddStackLabel(namespace, nil),
},
},
EndpointSpec: &swarm.EndpointSpec{
+4 -8
View File
@@ -9,9 +9,9 @@ import (
"golang.org/x/net/context"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/cli"
"github.com/docker/docker/cli/command"
"github.com/docker/docker/cli/compose/convert"
"github.com/docker/docker/client"
"github.com/spf13/cobra"
)
@@ -81,23 +81,19 @@ func getStacks(
ctx context.Context,
apiclient client.APIClient,
) ([]*stack, error) {
filter := filters.NewArgs()
filter.Add("label", labelNamespace)
services, err := apiclient.ServiceList(
ctx,
types.ServiceListOptions{Filters: filter})
types.ServiceListOptions{Filters: getAllStacksFilter()})
if err != nil {
return nil, err
}
m := make(map[string]*stack, 0)
for _, service := range services {
labels := service.Spec.Labels
name, ok := labels[labelNamespace]
name, ok := labels[convert.LabelNamespace]
if !ok {
return nil, fmt.Errorf("cannot get label %s for service %s",
labelNamespace, service.ID)
convert.LabelNamespace, service.ID)
}
ztack, ok := m[name]
if !ok {
+1 -2
View File
@@ -49,8 +49,7 @@ func runPS(dockerCli *command.DockerCli, opts psOptions) error {
client := dockerCli.Client()
ctx := context.Background()
filter := opts.filter.Value()
filter.Add("label", labelNamespace+"="+opts.namespace)
filter := getStackFilterFromOpt(opts.namespace, opts.filter)
if !opts.all && !filter.Include("desired-state") {
filter.Add("desired-state", string(swarm.TaskStateRunning))
filter.Add("desired-state", string(swarm.TaskStateAccepted))
+1 -3
View File
@@ -43,9 +43,7 @@ func runServices(dockerCli *command.DockerCli, opts servicesOptions) error {
ctx := context.Background()
client := dockerCli.Client()
filter := opts.filter.Value()
filter.Add("label", labelNamespace+"="+opts.namespace)
filter := getStackFilterFromOpt(opts.namespace, opts.filter)
services, err := client.ServiceList(ctx, types.ServiceListOptions{Filters: filter})
if err != nil {
return err