diff --git a/components/engine/.mailmap b/components/engine/.mailmap index 1078d5456c..4dc2d65f04 100644 --- a/components/engine/.mailmap +++ b/components/engine/.mailmap @@ -56,6 +56,7 @@ Benoit Chesneau Bhiraj Butala Bhumika Bayani Bilal Amarni +Bill Wang Bin Liu Bin Liu Bingshen Wang @@ -92,6 +93,7 @@ Daniel Dao Daniel Dao Daniel Garcia Daniel Gasienica +Daniel Goosen Daniel Grunwell Daniel J Walsh Daniel Mizyrycki @@ -99,14 +101,17 @@ Daniel Mizyrycki Daniel Mizyrycki Daniel Nephin Daniel Norberg +Daniel Watkins Danny Yates Darren Shepherd Dattatraya Kumbhar +Dave Goodchild Dave Henderson Dave Tucker David M. Karr David Sheets David Sissitka +David Williamson Deshi Xiao Deshi Xiao Diego Siqueira @@ -118,15 +123,18 @@ Elan Ruusamäe Elan Ruusamäe Eric G. Noriega Eric Hanchrow +Eric Rosenberg Erica Windisch Erica Windisch Erik Hollensbe Erwin van der Koogh Euan Kemp Eugen Krizo +Evan Hazlett Evelyn Xu Evgeny Shmarnev Faiz Khan +Felix Hupfeld Felix Ruess Feng Yan Fengtu Wang @@ -134,6 +142,7 @@ Francisco Carriedo Frank Rosquin Frederick F. Kautz IV Gabriel Nicolas Avellaneda +Gaetan de Villele Gang Qiao <1373319223@qq.com> George Kontridze Gerwim Feiken @@ -148,6 +157,7 @@ Guillaume J. Charmes Guillaume J. Charmes Gurjeet Singh Gustav Sinder +Günther Jungbluth Hakan Özler Hao Shu Wei Hao Shu Wei @@ -166,9 +176,12 @@ Hu Keping Huu Nguyen Hyzhou Zhy Hyzhou Zhy <1187766782@qq.com> +Ilya Khlopotov Jack Laxson Jacob Atzen Jacob Tomlinson +Jaivish Kothari +Jamie Hannaford Jean-Baptiste Barth Jean-Baptiste Dalido Jean-Tiare Le Bigot @@ -185,6 +198,7 @@ Jessica Frazelle Jessica Frazelle Jim Galasyn Jiuyue Ma +Joey Geiger Joffrey F Joffrey F Joffrey F @@ -200,6 +214,7 @@ Jordan Arentsen Jordan Jennings Jorit Kleine-Möllhoff Jose Diaz-Gonzalez +Josh Bonczkowski Josh Eveleth Josh Hawn Josh Horwitz @@ -225,11 +240,14 @@ Ken Herner Kenfe-Mickaël Laventure Kevin Feyrer Kevin Kern +Kevin Meredith Kir Kolyshkin Kir Kolyshkin +Kir Kolyshkin Konrad Kleine Konstantin Gribov Konstantin Pelykh +Kotaro Yoshimatsu Kunal Kushwaha Lajos Papp Lei Jitang @@ -243,6 +261,7 @@ Lokesh Mandvekar Lorenzo Fontana Louis Opter Louis Opter +Luca Favatella Luke Marsden Lyn Lynda O'Leary @@ -257,6 +276,7 @@ Marc Abramowitz Marcelo Horacio Fortino Marcus Linke Marianna Tessel +Mark Oates Markan Patel Markus Kortlang Martin Redmond @@ -281,6 +301,7 @@ Michael Huettermann Michael Käufl Michael Spetsiotis Michal Minář +Miguel Angel Alvarez Cabrerizo <30386061+doncicuto@users.noreply.github.com> Miguel Angel Fernández Mihai Borobocea Mike Casas @@ -296,6 +317,7 @@ Nathan LeClaire Nathan LeClaire Neil Horman Nick Russo +Nicolas Borboën Nigel Poulton Nik Nyby Nolan Darilek @@ -310,18 +332,23 @@ Pawel Konczalski Peter Choi Peter Dave Hello Peter Jaffe +Peter Nagy Peter Waller Phil Estes Philip Alexander Etling Philipp Gillé Qiang Huang Qiang Huang +Ray Tsang Renaud Gaubert Robert Terhaar Roberto G. Hashioka Roberto Muñoz Fernández Roman Dudin +Ross Boucher Runshen Zhu +Ryan Stelly +Sakeven Jiang Sandeep Bansal Sandeep Bansal Sargun Dhillon @@ -337,6 +364,7 @@ Shishir Mahajan Shukui Yang Shuwei Hao Shuwei Hao +Sidhartha Mani Sjoerd Langkemper Solomon Hykes Solomon Hykes @@ -355,6 +383,8 @@ Stephen Day Stephen Day Steve Desmond Sun Gengze <690388648@qq.com> +Sun Jianbo +Sun Jianbo Sven Dowideit Sven Dowideit Sven Dowideit @@ -387,6 +417,7 @@ Tõnis Tiigi Trishna Guha Tristan Carel Tristan Carel +Umesh Yadav Umesh Yadav Victor Lyuboslavsky Victor Vieux @@ -408,6 +439,7 @@ Walter Stanish Wang Guoliang Wang Jie Wang Ping +Wang Xing Wang Yuexiao Wayne Chang Wayne Song @@ -437,5 +469,6 @@ Zachary Jaffee Zachary Jaffee ZhangHang Zhenkun Bi +Zhou Hao Zhu Kunjia Zou Yu diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index afafb2d729..84059b7c2e 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -21,6 +21,7 @@ Adam Eijdenberg Adam Kunk Adam Miller Adam Mills +Adam Pointer Adam Singer Adam Walz Addam Hardy @@ -204,7 +205,7 @@ Bharath Thiruveedula Bhiraj Butala Bhumika Bayani Bilal Amarni -Bill W +Bill Wang Bin Liu Bingshen Wang Blake Geno @@ -212,7 +213,6 @@ Boaz Shuster bobby abbott Boris Pruessmann Boshi Lian -boucher Bouke Haarsma Boyd Hemphill boynux @@ -249,7 +249,6 @@ Bryan Bess Bryan Boreham Bryan Matsuo Bryan Murphy -buddhamagnet Burke Libbey Byung Kang Caleb Spare @@ -274,6 +273,7 @@ Cezar Sa Espinola Chad Swenson Chance Zibolski Chander Govindarajan +Chanhun Jeong Chao Wang Charles Chan Charles Hooper @@ -391,6 +391,7 @@ Daniel Nordberg Daniel Robinson Daniel S Daniel Von Fange +Daniel Watkins Daniel X Moore Daniel YC Lin Daniel Zhang @@ -403,6 +404,7 @@ Darren Stahl Dattatraya Kumbhar Davanum Srinivas Dave Barboza +Dave Goodchild Dave Henderson Dave MacDonald Dave Tucker @@ -429,7 +431,7 @@ David Röthlisberger David Sheets David Sissitka David Trott -David Williamson +David Williamson David Xia David Young Davide Ceretti @@ -498,6 +500,7 @@ Dr Nic Williams dragon788 Dražen Lučanin Drew Erny +Drew Hubl Dustin Sallings Ed Costello Edmund Wagner @@ -513,6 +516,7 @@ Elias Probst Elijah Zupancic eluck Elvir Kuric +Emil Davtyan Emil Hernvall Emily Maier Emily Rose @@ -528,7 +532,7 @@ Eric Lee Eric Myhre Eric Paris Eric Rafaloff -Eric Rosenberg +Eric Rosenberg Eric Sage Eric Soderstrom Eric Yang @@ -548,7 +552,6 @@ Eugen Krizo Eugene Yakubovich Evan Allrich Evan Carmi -Evan Hazlett Evan Hazlett Evan Krall Evan Phoenix @@ -578,7 +581,7 @@ Federico Gimenez Felipe Oliveira Felix Abecassis Felix Geisendörfer -Felix Hupfeld +Felix Hupfeld Felix Rabe Felix Ruess Felix Schindler @@ -590,10 +593,8 @@ Fero Volar Ferran Rodenas Filipe Brandenburger Filipe Oliveira -fl0yd Flavio Castelli Flavio Crisciani -FLGMwt Florian Florian Klein Florian Maier @@ -729,7 +730,7 @@ Iliana Weller Ilkka Laukkanen Ilya Dmitrichenko Ilya Gusev -ILYA Khlopotov +Ilya Khlopotov imre Fitos inglesp Ingo Gottwald @@ -749,6 +750,7 @@ Jacob Edelman Jacob Tomlinson Jacob Vallejo Jacob Wen +Jaivish Kothari Jake Champlin Jake Moshenko Jake Sanders @@ -765,7 +767,7 @@ James Mills James Nesbitt James Nugent James Turnbull -Jamie Hannaford +Jamie Hannaford Jamshid Afshar Jan Keromnes Jan Koprowski @@ -775,7 +777,6 @@ Jan-Gerd Tenberge Jan-Jaap Driessen Jana Radhakrishnan Jannick Fahlbusch -Janonymous Januar Wayong Jared Biel Jared Hocutt @@ -829,11 +830,9 @@ Jesse Dearing Jesse Dubay Jessica Frazelle Jezeniel Zapanta -jgeiger Jhon Honce Ji.Zhilong Jian Zhang -jianbosun Jie Luo Jihyun Hwang Jilles Oldenbeuving @@ -862,6 +861,7 @@ Joel Friedly Joel Handwell Joel Hansson Joel Wurtz +Joey Geiger Joey Geiger Joey Gibson Joffrey F @@ -914,6 +914,7 @@ Joseph Kern Joseph Rothrock Josh Josh Bodah +Josh Bonczkowski Josh Chorlton Josh Eveleth Josh Hawn @@ -986,12 +987,12 @@ Kevin J. Lynagh Kevin Jing Qiu Kevin Kern Kevin Menard +Kevin Meredith Kevin P. Kucharczyk Kevin Richardson Kevin Shi Kevin Wallace Kevin Yap -kevinmeredith Keyvan Fatehi kies Kim BKC Carlbacker @@ -999,7 +1000,6 @@ Kim Eik Kimbro Staken Kir Kolyshkin Kiran Gangadharan -Kirill Kolyshkin Kirill SIbirev knappe Kohei Tsuruta @@ -1074,7 +1074,7 @@ longliqiang88 <394564827@qq.com> Lorenz Leutgeb Lorenzo Fontana Louis Opter -Luca Favatella +Luca Favatella Luca Marturana Luca Orlandi Luca-Bogdan Grigorescu @@ -1131,6 +1131,7 @@ Mark Allen Mark McGranaghan Mark McKinstry Mark Milstein +Mark Oates Mark Parker Mark West Markan Patel @@ -1300,7 +1301,7 @@ Nick Stenning Nick Stinemates NickrenREN Nicola Kabar -Nicolas Borboën +Nicolas Borboën Nicolas De Loof Nicolas Dudebout Nicolas Goy @@ -1324,7 +1325,6 @@ Nuutti Kotivuori nzwsch O.S. Tezer objectified -OddBloke odk- Oguz Bilgic Oh Jinkyun @@ -1441,7 +1441,7 @@ Ralph Bean Ramkumar Ramachandra Ramon Brooker Ramon van Alteren -Ray Tsang +Ray Tsang ReadmeCritic Recursive Madman Reficul @@ -1500,7 +1500,6 @@ Roman Strashkin Ron Smits Ron Williams root -root root root root @@ -1524,6 +1523,7 @@ Ryan McLaughlin Ryan O'Donnell Ryan Seto Ryan Simmen +Ryan Stelly Ryan Thomas Ryan Trauntvein Ryan Wallner @@ -1537,7 +1537,7 @@ Sabin Basyal Sachin Joshi Sagar Hani Sainath Grandhi -sakeven +Sakeven Jiang Sally O'Malley Sam Abed Sam Alba @@ -1609,6 +1609,7 @@ shuai-z Shukui Yang Shuwei Hao Sian Lerk Lau +Sidhartha Mani sidharthamani Silas Sewell Silvan Jegen @@ -1660,6 +1661,7 @@ Steven Taylor Subhajit Ghosh Sujith Haridasan Sun Gengze <690388648@qq.com> +Sun Jianbo Sunny Gogoi Suryakumar Sudar Sven Dowideit @@ -1816,6 +1818,7 @@ Vladimir Pouzanov Vladimir Rutsky Vladimir Varankin VladimirAus +Vlastimil Zeman Vojtech Vitek (V-Teq) waitingkuo Walter Leibbrandt @@ -1856,9 +1859,7 @@ William Martin William Riancho William Thurston WiseTrem -wlan0 Wolfgang Powisch -wonderflow Wonjun Kim xamyzhao Xianglin Gao @@ -1921,7 +1922,7 @@ zhangxianwei Zhenan Ye <21551168@zju.edu.cn> zhenghenghuo Zhenkun Bi -zhouhao +Zhou Hao Zhu Guihua Zhu Kunjia Zhuoyun Wei diff --git a/components/engine/api/server/router/network/network_routes.go b/components/engine/api/server/router/network/network_routes.go index 450d73b4fe..df7af31997 100644 --- a/components/engine/api/server/router/network/network_routes.go +++ b/components/engine/api/server/router/network/network_routes.go @@ -164,6 +164,13 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r // return the network. Skipped using isMatchingScope because it is true if the scope // is not set which would be case if the client API v1.30 if strings.HasPrefix(nwk.ID, term) || (netconst.SwarmScope == scope) { + // If we have a previous match "backend", return it, we need verbose when enabled + // ex: overlay/partial_ID or name/swarm_scope + if nwv, ok := listByPartialID[nwk.ID]; ok { + nwk = nwv + } else if nwv, ok := listByFullName[nwk.ID]; ok { + nwk = nwv + } return httputils.WriteJSON(w, http.StatusOK, nwk) } } diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 4ba5ad9e27..0f52f3c3fb 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -322,8 +322,11 @@ func verifyNetworkingConfig(nwConfig *networktypes.NetworkingConfig) error { return nil } if len(nwConfig.EndpointsConfig) == 1 { - for _, v := range nwConfig.EndpointsConfig { - if v != nil && v.IPAMConfig != nil { + for k, v := range nwConfig.EndpointsConfig { + if v == nil { + return errdefs.InvalidParameter(errors.Errorf("no EndpointSettings for %s", k)) + } + if v.IPAMConfig != nil { if v.IPAMConfig.IPv4Address != "" && net.ParseIP(v.IPAMConfig.IPv4Address).To4() == nil { return errors.Errorf("invalid IPv4 address: %s", v.IPAMConfig.IPv4Address) } diff --git a/components/engine/daemon/create_test.go b/components/engine/daemon/create_test.go new file mode 100644 index 0000000000..43ac5b390a --- /dev/null +++ b/components/engine/daemon/create_test.go @@ -0,0 +1,21 @@ +package daemon + +import ( + "testing" + + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/errdefs" + "github.com/stretchr/testify/assert" +) + +// Test case for 35752 +func TestVerifyNetworkingConfig(t *testing.T) { + name := "mynet" + endpoints := make(map[string]*network.EndpointSettings, 1) + endpoints[name] = nil + nwConfig := &network.NetworkingConfig{ + EndpointsConfig: endpoints, + } + err := verifyNetworkingConfig(nwConfig) + assert.True(t, errdefs.IsInvalidParameter(err)) +} diff --git a/components/engine/daemon/daemon_unix.go b/components/engine/daemon/daemon_unix.go index 74046147a1..de9537c1b0 100644 --- a/components/engine/daemon/daemon_unix.go +++ b/components/engine/daemon/daemon_unix.go @@ -28,6 +28,7 @@ import ( "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" + "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/sysinfo" @@ -1169,6 +1170,12 @@ func setupDaemonRoot(config *config.Config, rootDir string, rootIDs idtools.IDPa } } } + + if err := ensureSharedOrSlave(config.Root); err != nil { + if err := mount.MakeShared(config.Root); err != nil { + logrus.WithError(err).WithField("dir", config.Root).Warn("Could not set daemon root propagation to shared, this is not generally critical but may cause some functionality to not work or fallback to less desirable behavior") + } + } return nil } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 626de7767a..28ad2d912a 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -303,7 +303,7 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis var beforeContFilter, sinceContFilter *container.Snapshot err = psFilters.WalkValues("before", func(value string) error { - beforeContFilter, err = view.Get(value) + beforeContFilter, err = idOrNameFilter(view, value) return err }) if err != nil { @@ -311,7 +311,7 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis } err = psFilters.WalkValues("since", func(value string) error { - sinceContFilter, err = view.Get(value) + sinceContFilter, err = idOrNameFilter(view, value) return err }) if err != nil { @@ -365,6 +365,30 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis names: view.GetAllNames(), }, nil } + +func idOrNameFilter(view container.View, value string) (*container.Snapshot, error) { + filter, err := view.Get(value) + switch err.(type) { + case container.NoSuchContainerError: + // Try name search instead + found := "" + for id, idNames := range view.GetAllNames() { + for _, eachName := range idNames { + if strings.TrimPrefix(value, "/") == strings.TrimPrefix(eachName, "/") { + if found != "" && found != id { + return nil, err + } + found = id + } + } + } + if found != "" { + filter, err = view.Get(found) + } + } + return filter, err +} + func portOp(key string, filter map[nat.Port]bool) func(value string) error { return func(value string) error { if strings.Contains(value, ":") { diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go index 25dd2152f0..d3ebeb6999 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs.go @@ -410,7 +410,7 @@ func (l *logStream) collectBatch() { // If event buffer is older than batch publish frequency flush the event buffer if eventBufferTimestamp > 0 && len(eventBuffer) > 0 { eventBufferAge := t.UnixNano()/int64(time.Millisecond) - eventBufferTimestamp - eventBufferExpired := eventBufferAge > int64(batchPublishFrequency)/int64(time.Millisecond) + eventBufferExpired := eventBufferAge >= int64(batchPublishFrequency)/int64(time.Millisecond) eventBufferNegative := eventBufferAge < 0 if eventBufferExpired || eventBufferNegative { l.processEvent(batch, eventBuffer, eventBufferTimestamp) @@ -431,21 +431,23 @@ func (l *logStream) collectBatch() { if eventBufferTimestamp == 0 { eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) } - unprocessedLine := msg.Line + line := msg.Line if l.multilinePattern != nil { - if l.multilinePattern.Match(unprocessedLine) || len(eventBuffer)+len(unprocessedLine) > maximumBytesPerEvent { + if l.multilinePattern.Match(line) || len(eventBuffer)+len(line) > maximumBytesPerEvent { // This is a new log event or we will exceed max bytes per event // so flush the current eventBuffer to events and reset timestamp l.processEvent(batch, eventBuffer, eventBufferTimestamp) eventBufferTimestamp = msg.Timestamp.UnixNano() / int64(time.Millisecond) eventBuffer = eventBuffer[:0] } - // Append new line - processedLine := append(unprocessedLine, "\n"...) - eventBuffer = append(eventBuffer, processedLine...) + // Append new line if event is less than max event size + if len(line) < maximumBytesPerEvent { + line = append(line, "\n"...) + } + eventBuffer = append(eventBuffer, line...) logger.PutMessage(msg) } else { - l.processEvent(batch, unprocessedLine, msg.Timestamp.UnixNano()/int64(time.Millisecond)) + l.processEvent(batch, line, msg.Timestamp.UnixNano()/int64(time.Millisecond)) logger.PutMessage(msg) } } @@ -461,14 +463,14 @@ func (l *logStream) collectBatch() { // bytes per event (defined in maximumBytesPerEvent). There is a fixed per-event // byte overhead (defined in perEventBytes) which is accounted for in split- and // batch-calculations. -func (l *logStream) processEvent(batch *eventBatch, unprocessedLine []byte, timestamp int64) { - for len(unprocessedLine) > 0 { +func (l *logStream) processEvent(batch *eventBatch, events []byte, timestamp int64) { + for len(events) > 0 { // Split line length so it does not exceed the maximum - lineBytes := len(unprocessedLine) + lineBytes := len(events) if lineBytes > maximumBytesPerEvent { lineBytes = maximumBytesPerEvent } - line := unprocessedLine[:lineBytes] + line := events[:lineBytes] event := wrappedEvent{ inputLogEvent: &cloudwatchlogs.InputLogEvent{ @@ -480,7 +482,7 @@ func (l *logStream) processEvent(batch *eventBatch, unprocessedLine []byte, time added := batch.add(event, lineBytes) if added { - unprocessedLine = unprocessedLine[lineBytes:] + events = events[lineBytes:] } else { l.publishBatch(batch) batch.reset() diff --git a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go index 67ea474767..bf5d6dd29e 100644 --- a/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go +++ b/components/engine/daemon/logger/awslogs/cloudwatchlogs_test.go @@ -726,6 +726,59 @@ func TestCollectBatchMultilinePatternNegativeEventAge(t *testing.T) { stream.Close() } +func TestCollectBatchMultilinePatternMaxEventSize(t *testing.T) { + mockClient := newMockClient() + multilinePattern := regexp.MustCompile("xxxx") + stream := &logStream{ + client: mockClient, + logGroupName: groupName, + logStreamName: streamName, + multilinePattern: multilinePattern, + sequenceToken: aws.String(sequenceToken), + messages: make(chan *logger.Message), + } + mockClient.putLogEventsResult <- &putLogEventsResult{ + successResult: &cloudwatchlogs.PutLogEventsOutput{ + NextSequenceToken: aws.String(nextSequenceToken), + }, + } + ticks := make(chan time.Time) + newTicker = func(_ time.Duration) *time.Ticker { + return &time.Ticker{ + C: ticks, + } + } + + go stream.collectBatch() + + // Log max event size + longline := strings.Repeat("A", maximumBytesPerEvent) + stream.Log(&logger.Message{ + Line: []byte(longline), + Timestamp: time.Now(), + }) + + // Log short event + shortline := strings.Repeat("B", 100) + stream.Log(&logger.Message{ + Line: []byte(shortline), + Timestamp: time.Now(), + }) + + // Fire ticker + ticks <- time.Now().Add(batchPublishFrequency) + + // Verify multiline events + // We expect a maximum sized event with no new line characters and a + // second short event with a new line character at the end + argument := <-mockClient.putLogEventsArgument + assert.NotNil(t, argument, "Expected non-nil PutLogEventsInput") + assert.Equal(t, 2, len(argument.LogEvents), "Expected two events") + assert.Equal(t, longline, *argument.LogEvents[0].Message, "Received incorrect multiline message") + assert.Equal(t, shortline+"\n", *argument.LogEvents[1].Message, "Received incorrect multiline message") + stream.Close() +} + func TestCollectBatchClose(t *testing.T) { mockClient := newMockClient() stream := &logStream{ diff --git a/components/engine/integration/container/ps_test.go b/components/engine/integration/container/ps_test.go new file mode 100644 index 0000000000..b7eaa72f44 --- /dev/null +++ b/components/engine/integration/container/ps_test.go @@ -0,0 +1,64 @@ +package container + +import ( + "context" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/util/request" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPsFilter(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + createContainerForFilter := func(ctx context.Context, name string) string { + body, err := client.ContainerCreate(ctx, + &container.Config{ + Cmd: []string{"top"}, + Image: "busybox", + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + name, + ) + require.NoError(t, err) + return body.ID + } + + prev := createContainerForFilter(ctx, "prev") + createContainerForFilter(ctx, "top") + next := createContainerForFilter(ctx, "next") + + containerIDs := func(containers []types.Container) []string { + entries := []string{} + for _, container := range containers { + entries = append(entries, container.ID) + } + return entries + } + + f1 := filters.NewArgs() + f1.Add("since", "top") + q1, err := client.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Filters: f1, + }) + require.NoError(t, err) + assert.Contains(t, containerIDs(q1), next) + + f2 := filters.NewArgs() + f2.Add("before", "top") + q2, err := client.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Filters: f2, + }) + require.NoError(t, err) + assert.Contains(t, containerIDs(q2), prev) +} diff --git a/components/engine/integration/network/inspect_test.go b/components/engine/integration/network/inspect_test.go new file mode 100644 index 0000000000..8f8c846a9c --- /dev/null +++ b/components/engine/integration/network/inspect_test.go @@ -0,0 +1,221 @@ +package network + +import ( + "fmt" + "runtime" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/client" + "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration-cli/request" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +const defaultSwarmPort = 2477 +const dockerdBinary = "dockerd" + +func TestInspectNetwork(t *testing.T) { + defer setupTest(t)() + d := newSwarm(t) + defer d.Stop(t) + client, err := request.NewClientForHost(d.Sock()) + require.NoError(t, err) + + overlayName := "overlay1" + networkCreate := types.NetworkCreate{ + CheckDuplicate: true, + Driver: "overlay", + } + + netResp, err := client.NetworkCreate(context.Background(), overlayName, networkCreate) + require.NoError(t, err) + overlayID := netResp.ID + + var instances uint64 = 4 + serviceName := "TestService" + serviceSpec := swarmServiceSpec(serviceName, instances) + serviceSpec.TaskTemplate.Networks = append(serviceSpec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{Target: overlayName}) + + serviceResp, err := client.ServiceCreate(context.Background(), serviceSpec, types.ServiceCreateOptions{ + QueryRegistry: false, + }) + require.NoError(t, err) + + pollSettings := func(config *poll.Settings) { + if runtime.GOARCH == "arm" { + config.Timeout = 30 * time.Second + config.Delay = 100 * time.Millisecond + } + } + + serviceID := serviceResp.ID + poll.WaitOn(t, serviceRunningTasksCount(client, serviceID, instances), pollSettings) + + _, _, err = client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{}) + require.NoError(t, err) + + // Test inspect verbose with full NetworkID + networkVerbose, err := client.NetworkInspect(context.Background(), overlayID, types.NetworkInspectOptions{ + Verbose: true, + }) + require.NoError(t, err) + require.True(t, validNetworkVerbose(networkVerbose, serviceName, instances)) + + // Test inspect verbose with partial NetworkID + networkVerbose, err = client.NetworkInspect(context.Background(), overlayID[0:11], types.NetworkInspectOptions{ + Verbose: true, + }) + require.NoError(t, err) + require.True(t, validNetworkVerbose(networkVerbose, serviceName, instances)) + + // Test inspect verbose with Network name and swarm scope + networkVerbose, err = client.NetworkInspect(context.Background(), overlayName, types.NetworkInspectOptions{ + Verbose: true, + Scope: "swarm", + }) + require.NoError(t, err) + require.True(t, validNetworkVerbose(networkVerbose, serviceName, instances)) + + err = client.ServiceRemove(context.Background(), serviceID) + require.NoError(t, err) + + poll.WaitOn(t, serviceIsRemoved(client, serviceID), pollSettings) + poll.WaitOn(t, noTasks(client), pollSettings) + + serviceResp, err = client.ServiceCreate(context.Background(), serviceSpec, types.ServiceCreateOptions{ + QueryRegistry: false, + }) + require.NoError(t, err) + + serviceID2 := serviceResp.ID + poll.WaitOn(t, serviceRunningTasksCount(client, serviceID2, instances), pollSettings) + + err = client.ServiceRemove(context.Background(), serviceID2) + require.NoError(t, err) + + poll.WaitOn(t, serviceIsRemoved(client, serviceID2), pollSettings) + poll.WaitOn(t, noTasks(client), pollSettings) + + err = client.NetworkRemove(context.Background(), overlayID) + require.NoError(t, err) + + poll.WaitOn(t, networkIsRemoved(client, overlayID), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second)) +} + +func newSwarm(t *testing.T) *daemon.Swarm { + d := &daemon.Swarm{ + Daemon: daemon.New(t, "", dockerdBinary, daemon.Config{ + Experimental: testEnv.DaemonInfo.ExperimentalBuild, + }), + // TODO: better method of finding an unused port + Port: defaultSwarmPort, + } + // TODO: move to a NewSwarm constructor + d.ListenAddr = fmt.Sprintf("0.0.0.0:%d", d.Port) + + // avoid networking conflicts + args := []string{"--iptables=false", "--swarm-default-advertise-addr=lo"} + d.StartWithBusybox(t, args...) + + require.NoError(t, d.Init(swarm.InitRequest{})) + return d +} + +func swarmServiceSpec(name string, replicas uint64) swarm.ServiceSpec { + return swarm.ServiceSpec{ + Annotations: swarm.Annotations{ + Name: name, + }, + TaskTemplate: swarm.TaskSpec{ + ContainerSpec: &swarm.ContainerSpec{ + Image: "busybox:latest", + Command: []string{"/bin/top"}, + }, + }, + Mode: swarm.ServiceMode{ + Replicated: &swarm.ReplicatedService{ + Replicas: &replicas, + }, + }, + } +} + +func serviceRunningTasksCount(client client.ServiceAPIClient, serviceID string, instances uint64) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + filter := filters.NewArgs() + filter.Add("service", serviceID) + tasks, err := client.TaskList(context.Background(), types.TaskListOptions{ + Filters: filter, + }) + switch { + case err != nil: + return poll.Error(err) + case len(tasks) == int(instances): + for _, task := range tasks { + if task.Status.State != swarm.TaskStateRunning { + return poll.Continue("waiting for tasks to enter run state") + } + } + return poll.Success() + default: + return poll.Continue("task count at %d waiting for %d", len(tasks), instances) + } + } +} + +func networkIsRemoved(client client.NetworkAPIClient, networkID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + _, err := client.NetworkInspect(context.Background(), networkID, types.NetworkInspectOptions{}) + if err == nil { + return poll.Continue("waiting for network %s to be removed", networkID) + } + return poll.Success() + } +} + +func serviceIsRemoved(client client.ServiceAPIClient, serviceID string) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + filter := filters.NewArgs() + filter.Add("service", serviceID) + _, err := client.TaskList(context.Background(), types.TaskListOptions{ + Filters: filter, + }) + if err == nil { + return poll.Continue("waiting for service %s to be deleted", serviceID) + } + return poll.Success() + } +} + +func noTasks(client client.ServiceAPIClient) func(log poll.LogT) poll.Result { + return func(log poll.LogT) poll.Result { + filter := filters.NewArgs() + tasks, err := client.TaskList(context.Background(), types.TaskListOptions{ + Filters: filter, + }) + switch { + case err != nil: + return poll.Error(err) + case len(tasks) == 0: + return poll.Success() + default: + return poll.Continue("task count at %d waiting for 0", len(tasks)) + } + } +} + +// Check to see if Service and Tasks info are part of the inspect verbose response +func validNetworkVerbose(network types.NetworkResource, service string, instances uint64) bool { + if service, ok := network.Services[service]; ok { + if len(service.Tasks) == int(instances) { + return true + } + } + return false +} diff --git a/components/engine/pkg/mount/mount.go b/components/engine/pkg/mount/mount.go index ee5833c49d..0d02a575f6 100644 --- a/components/engine/pkg/mount/mount.go +++ b/components/engine/pkg/mount/mount.go @@ -4,6 +4,8 @@ import ( "sort" "strings" + "syscall" + "github.com/sirupsen/logrus" ) @@ -77,18 +79,30 @@ func RecursiveUnmount(target string) error { continue } logrus.Debugf("Trying to unmount %s", m.Mountpoint) - err = Unmount(m.Mountpoint) - if err != nil && i == len(mounts)-1 { - if mounted, err := Mounted(m.Mountpoint); err != nil || mounted { - return err + err = unmount(m.Mountpoint, mntDetach) + if err != nil { + // If the error is EINVAL either this whole package is wrong (invalid flags passed to unmount(2)) or this is + // not a mountpoint (which is ok in this case). + // Meanwhile calling `Mounted()` is very expensive. + // + // We've purposefully used `syscall.EINVAL` here instead of `unix.EINVAL` to avoid platform branching + // Since `EINVAL` is defined for both Windows and Linux in the `syscall` package (and other platforms), + // this is nicer than defining a custom value that we can refer to in each platform file. + if err == syscall.EINVAL { + continue } - // Ignore errors for submounts and continue trying to unmount others - // The final unmount should fail if there ane any submounts remaining - } else if err != nil { - logrus.Errorf("Failed to unmount %s: %v", m.Mountpoint, err) - } else if err == nil { - logrus.Debugf("Unmounted %s", m.Mountpoint) + if i == len(mounts)-1 { + if mounted, e := Mounted(m.Mountpoint); e != nil || mounted { + return err + } + continue + } + // This is some submount, we can ignore this error for now, the final unmount will fail if this is a real problem + logrus.WithError(err).Warnf("Failed to unmount submount %s", m.Mountpoint) + continue } + + logrus.Debugf("Unmounted %s", m.Mountpoint) } return nil }