From 66b0c6b9463589b5226eaf855da9d9576c51b51d Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 9 Sep 2013 18:14:23 -0700 Subject: [PATCH 1/5] Split stdout stderr for non-tty commands Upstream-commit: 8ae5348a51ab6026db29009f109bb3724908f0aa Component: engine --- components/engine/api_test.go | 5 +- components/engine/commands.go | 32 ++++++++---- components/engine/server.go | 12 ++++- components/engine/utils/utils.go | 87 ++++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 16 deletions(-) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 837a20c244..fd736d5c98 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -445,7 +445,7 @@ func TestGetContainersChanges(t *testing.T) { } func TestGetContainersTop(t *testing.T) { - t.Skip("Fixme. Skipping test for now. Reported error when testing using dind: 'api_test.go:527: Expected 2 processes, found 0.'") + t.Skip("Fixme. Skipping test for now. Reported error when testing using dind: 'api_test.go:527: Expected 2 processes, found 0.'") runtime, err := newTestRuntime() if err != nil { t.Fatal(err) @@ -566,7 +566,6 @@ func TestPostCommit(t *testing.T) { srv := &Server{runtime: runtime} - // Create a container and remove a file container, err := runtime.Create( &Config{ @@ -952,7 +951,7 @@ func TestPostContainersAttach(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string(utils.Stdout)+"hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) } }) diff --git a/components/engine/commands.go b/components/engine/commands.go index 00eeaf3881..7f6f4cc0b5 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1219,7 +1219,7 @@ func (cli *DockerCli) CmdLogs(args ...string) error { return nil } - if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1&stderr=1", false, nil, cli.out); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?logs=1&stdout=1&stderr=1", false, nil, cli.out, cli.err); err != nil { return err } return nil @@ -1262,7 +1262,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { v.Set("stdout", "1") v.Set("stderr", "1") - if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, cli.in, cli.out); err != nil { + if err := cli.hijack("POST", "/containers/"+cmd.Arg(0)+"/attach?"+v.Encode(), container.Config.Tty, cli.in, cli.out, cli.err); err != nil { return err } return nil @@ -1526,7 +1526,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { v := url.Values{} v.Set("logs", "1") v.Set("stream", "1") - var out io.Writer + var out, stderr io.Writer if config.AttachStdin { v.Set("stdin", "1") @@ -1537,7 +1537,11 @@ func (cli *DockerCli) CmdRun(args ...string) error { } if config.AttachStderr { v.Set("stderr", "1") - out = cli.out + if config.Tty { + stderr = cli.out + } else { + stderr = cli.err + } } signals := make(chan os.Signal, 1) @@ -1551,7 +1555,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { } }() - if err := cli.hijack("POST", "/containers/"+runResult.ID+"/attach?"+v.Encode(), config.Tty, cli.in, out); err != nil { + if err := cli.hijack("POST", "/containers/"+runResult.ID+"/attach?"+v.Encode(), config.Tty, cli.in, out, stderr); err != nil { utils.Debugf("Error hijack: %s", err) return err } @@ -1718,7 +1722,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h return nil } -func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, out io.Writer) error { +func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.ReadCloser, stdout, stderr io.Writer) error { req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil) if err != nil { @@ -1744,10 +1748,16 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea rwc, br := clientconn.Hijack() defer rwc.Close() - var receiveStdout (chan error) - if out != nil { - receiveStdout = utils.Go(func() error { - _, err := io.Copy(out, br) + var receiveStdout chan error + + if stdout != nil { + receiveStdout = utils.Go(func() (err error) { + // When TTY is ON, use regular copy + if setRawTerminal { + _, err = io.Copy(stdout, br) + } else { + _, err = utils.StdCopy(stdout, stderr, br) + } utils.Debugf("[hijack] End of stdout") return err }) @@ -1779,7 +1789,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea return nil }) - if out != nil { + if stdout != nil { if err := <-receiveStdout; err != nil { utils.Debugf("Error receiveStdout: %s", err) return err diff --git a/components/engine/server.go b/components/engine/server.go index 322d8fe93e..c796c216e5 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1244,10 +1244,18 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std cStdinCloser = in } if stdout { - cStdout = out + if container.Config.Tty { + cStdout = out + } else { + cStdout = utils.NewStdWriter(out, utils.Stdout) + } } if stderr { - cStderr = out + if container.Config.Tty { + cStderr = out + } else { + cStderr = utils.NewStdWriter(out, utils.Stderr) + } } <-container.Attach(cStdin, cStdinCloser, cStdout, cStderr) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index d417690c0c..18b695a3eb 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -1021,3 +1021,90 @@ type StatusError struct { func (e *StatusError) Error() string { return fmt.Sprintf("Status: %d", e.Status) } + +type StdType []byte + +const StdWriterPrefixLen = 8 + +var ( + Stdin StdType = StdType("\001 stdin\002") + Stdout StdType = StdType("\001stdout\002") + Stderr StdType = StdType("\001stderr\002") +) + +type StdWriter struct { + io.Writer + prefix []byte +} + +func (w *StdWriter) Write(buf []byte) (n int, err error) { + n, err = w.Writer.Write(append(w.prefix, buf...)) + if n >= len(buf)+StdWriterPrefixLen { + n -= StdWriterPrefixLen + } + return n, err +} + +// NewStdWriter instanciate a new Writer based on the given type `t`. +// the utils package contains the valid parametres for `t`: +func NewStdWriter(w io.Writer, t StdType) *StdWriter { + if len(t) != StdWriterPrefixLen { + return nil + } + return &StdWriter{ + Writer: w, + prefix: []byte(t), + } +} + +// StdCopy is a modified version of io.Copy. +// +// StdCopy copies from src to dstout or dsterr until either EOF is reached +// on src or an error occurs. It returns the number of bytes +// copied and the first error encountered while copying, if any. +// +// A successful Copy returns err == nil, not err == EOF. +// Because Copy is defined to read from src until EOF, it does +// not treat an EOF from Read as an error to be reported. +// +// The source needs to be writter via StdWriter, dstout or dsterr is selected +// based on the prefix added by StdWriter +func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { + var ( + buf = make([]byte, 32*1024) + nw int + ew error + ) + + for { + nr, er := src.Read(buf) + if nr > 0 { + if bytes.Compare(buf[:StdWriterPrefixLen], Stdout) == 0 { + nw, ew = dstout.Write(buf[StdWriterPrefixLen:nr]) + } else if bytes.Compare(buf[:StdWriterPrefixLen], Stderr) == 0 { + nw, ew = dsterr.Write(buf[StdWriterPrefixLen:nr]) + } else if bytes.Compare(buf[:StdWriterPrefixLen], Stdin) == 0 { + nw, ew = dstout.Write(buf[StdWriterPrefixLen:nr]) + } + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr-StdWriterPrefixLen != nw { + err = io.ErrShortWrite + break + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return written, err +} From 72e85e9acdeba9e21e0a758181cbabe1efe73500 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 9 Sep 2013 18:23:03 -0700 Subject: [PATCH 2/5] Add stderr unit test Upstream-commit: 289350e63397bf918fbd75726a6e06f0fe49b81b Component: engine --- components/engine/api_test.go | 89 +++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index fd736d5c98..36de4e493c 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -979,6 +979,95 @@ func TestPostContainersAttach(t *testing.T) { container.Wait() } +func TestPostContainersAttachStderr(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + container, err := runtime.Create( + &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"/bin/sh", "-c", "/bin/cat >&2"}, + OpenStdin: true, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + // Start the process + hostConfig := &HostConfig{} + if err := container.Start(hostConfig); err != nil { + t.Fatal(err) + } + + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + // Try to avoid the timeout in destroy. Best effort, don't check error + defer func() { + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) + container.Kill() + }() + + // Attach to it + c1 := make(chan struct{}) + go func() { + defer close(c1) + + r := &hijackTester{ + ResponseRecorder: httptest.NewRecorder(), + in: stdin, + out: stdoutPipe, + } + + req, err := http.NewRequest("POST", "/containers/"+container.ID+"/attach?stream=1&stdin=1&stdout=1&stderr=1", bytes.NewReader([]byte{})) + if err != nil { + t.Fatal(err) + } + + if err := postContainersAttach(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + }() + + // Acknowledge hijack + setTimeout(t, "hijack acknowledge timed out", 2*time.Second, func() { + stdout.Read([]byte{}) + stdout.Read(make([]byte, 4096)) + }) + + setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", string(utils.Stderr)+"hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + // Close pipes (client disconnects) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // Wait for attach to finish, the client disconnected, therefore, Attach finished his job + setTimeout(t, "Waiting for CmdAttach timed out", 10*time.Second, func() { + <-c1 + }) + + // We closed stdin, expect /bin/cat to still be running + // Wait a little bit to make sure container.monitor() did his thing + err = container.WaitTimeout(500 * time.Millisecond) + if err == nil || !container.State.Running { + t.Fatalf("/bin/cat is not running after closing stdin") + } + + // Try to avoid the timeout in destroy. Best effort, don't check error + cStdin, _ := container.StdinPipe() + cStdin.Close() + container.Wait() +} + // FIXME: Test deleting running container // FIXME: Test deleting container with volume // FIXME: Test deleting volume in use by other container From eac5eb816d988786c678d58cff340f50ccd72de5 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 11 Sep 2013 11:35:09 -0700 Subject: [PATCH 3/5] Make StdCopy works with huge amount of data Upstream-commit: e854b7b2e6666d8b47f2c9714ac3bef2320fc40b Component: engine --- components/engine/api.go | 31 +++-- components/engine/api_test.go | 4 +- components/engine/commands.go | 2 +- components/engine/server.go | 25 ++-- components/engine/utils/stdcopy.go | 179 +++++++++++++++++++++++++++++ components/engine/utils/utils.go | 87 -------------- 6 files changed, 212 insertions(+), 116 deletions(-) create mode 100644 components/engine/utils/stdcopy.go diff --git a/components/engine/api.go b/components/engine/api.go index f4a96a0307..1b36221ba6 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -766,32 +766,43 @@ func postContainersAttach(srv *Server, version float64, w http.ResponseWriter, r } name := vars["name"] - if _, err := srv.ContainerInspect(name); err != nil { + c, err := srv.ContainerInspect(name) + if err != nil { return err } - in, out, err := hijackServer(w) + inStream, outStream, err := hijackServer(w) if err != nil { return err } defer func() { - if tcpc, ok := in.(*net.TCPConn); ok { + if tcpc, ok := inStream.(*net.TCPConn); ok { tcpc.CloseWrite() } else { - in.Close() + inStream.Close() } }() defer func() { - if tcpc, ok := out.(*net.TCPConn); ok { + if tcpc, ok := outStream.(*net.TCPConn); ok { tcpc.CloseWrite() - } else if closer, ok := out.(io.Closer); ok { + } else if closer, ok := outStream.(io.Closer); ok { closer.Close() } }() - fmt.Fprintf(out, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") - if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, in, out); err != nil { - fmt.Fprintf(out, "Error: %s\n", err) + var errStream io.Writer + + fmt.Fprintf(outStream, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") + + if !c.Config.Tty && version >= 1.4 { + errStream = utils.NewStdWriter(outStream, utils.Stderr) + outStream = utils.NewStdWriter(outStream, utils.Stdout) + } else { + errStream = outStream + } + + if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, inStream, outStream, errStream); err != nil { + fmt.Fprintf(outStream, "Error: %s\n", err) } return nil } @@ -834,7 +845,7 @@ func wsContainersAttach(srv *Server, version float64, w http.ResponseWriter, r * h := websocket.Handler(func(ws *websocket.Conn) { defer ws.Close() - if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, ws, ws); err != nil { + if err := srv.ContainerAttach(name, logs, stream, stdin, stdout, stderr, ws, ws, ws); err != nil { utils.Debugf("Error: %s", err) } }) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 36de4e493c..f652b07844 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -951,7 +951,7 @@ func TestPostContainersAttach(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string(utils.Stdout)+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{1, 0, 0, 0, 6, 0, 0, 0})+"hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) } }) @@ -1040,7 +1040,7 @@ func TestPostContainersAttachStderr(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string(utils.Stderr)+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{2, 0, 0, 0, 6, 0, 0, 0})+"hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) } }) diff --git a/components/engine/commands.go b/components/engine/commands.go index 7f6f4cc0b5..4c0436000b 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1570,7 +1570,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { return err } if status != 0 { - return &utils.StatusError{status} + return &utils.StatusError{Status: status} } } diff --git a/components/engine/server.go b/components/engine/server.go index c796c216e5..b40f60556e 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1175,11 +1175,12 @@ func (srv *Server) ContainerResize(name string, h, w int) error { return fmt.Errorf("No such container: %s", name) } -func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, stderr bool, in io.ReadCloser, out io.Writer) error { +func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, stderr bool, inStream io.ReadCloser, outStream, errStream io.Writer) error { container := srv.runtime.Get(name) if container == nil { return fmt.Errorf("No such container: %s", name) } + //logs if logs { cLog, err := container.ReadLog("json") @@ -1190,7 +1191,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std cLog, err := container.ReadLog("stdout") if err != nil { utils.Debugf("Error reading logs (stdout): %s", err) - } else if _, err := io.Copy(out, cLog); err != nil { + } else if _, err := io.Copy(outStream, cLog); err != nil { utils.Debugf("Error streaming logs (stdout): %s", err) } } @@ -1198,7 +1199,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std cLog, err := container.ReadLog("stderr") if err != nil { utils.Debugf("Error reading logs (stderr): %s", err) - } else if _, err := io.Copy(out, cLog); err != nil { + } else if _, err := io.Copy(errStream, cLog); err != nil { utils.Debugf("Error streaming logs (stderr): %s", err) } } @@ -1215,7 +1216,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std break } if (l.Stream == "stdout" && stdout) || (l.Stream == "stderr" && stderr) { - fmt.Fprintf(out, "%s", l.Log) + fmt.Fprintf(outStream, "%s", l.Log) } } } @@ -1238,24 +1239,16 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std go func() { defer w.Close() defer utils.Debugf("Closing buffered stdin pipe") - io.Copy(w, in) + io.Copy(w, inStream) }() cStdin = r - cStdinCloser = in + cStdinCloser = inStream } if stdout { - if container.Config.Tty { - cStdout = out - } else { - cStdout = utils.NewStdWriter(out, utils.Stdout) - } + cStdout = outStream } if stderr { - if container.Config.Tty { - cStderr = out - } else { - cStderr = utils.NewStdWriter(out, utils.Stderr) - } + cStderr = errStream } <-container.Attach(cStdin, cStdinCloser, cStdout, cStderr) diff --git a/components/engine/utils/stdcopy.go b/components/engine/utils/stdcopy.go new file mode 100644 index 0000000000..4965ea41b0 --- /dev/null +++ b/components/engine/utils/stdcopy.go @@ -0,0 +1,179 @@ +package utils + +import ( + "encoding/binary" + "errors" + "io" + "unsafe" +) + +func CheckBigEndian() bool { + var x uint32 = 0x01020304 + + if *(*byte)(unsafe.Pointer(&x)) == 0x01 { + return true + } + return false +} + +const ( + StdWriterPrefixLen = 8 + StdWriterFdIndex = 0 + StdWriterSizeIndex = 4 +) + +type StdType [StdWriterPrefixLen]byte + +var ( + Stdin StdType = StdType{0: 0} + Stdout StdType = StdType{0: 1} + Stderr StdType = StdType{0: 2} +) + +type StdWriter struct { + io.Writer + prefix StdType + sizeBuf []byte + byteOrder binary.ByteOrder +} + +func (w *StdWriter) Write(buf []byte) (n int, err error) { + if w == nil || w.Writer == nil { + return 0, errors.New("Writer not instanciated") + } + w.byteOrder.PutUint32(w.prefix[4:], uint32(len(buf))) + buf = append(w.prefix[:], buf...) + + n, err = w.Writer.Write(buf) + return n - StdWriterPrefixLen, err +} + +// NewStdWriter instanciate a new Writer based on the given type `t`. +// the utils package contains the valid parametres for `t`: +func NewStdWriter(w io.Writer, t StdType) *StdWriter { + if len(t) != StdWriterPrefixLen { + return nil + } + + var bo binary.ByteOrder + + if CheckBigEndian() { + bo = binary.BigEndian + } else { + bo = binary.LittleEndian + } + return &StdWriter{ + Writer: w, + prefix: t, + sizeBuf: make([]byte, 4), + byteOrder: bo, + } +} + +var ErrInvalidStdHeader = errors.New("Unrecognized input header") + +// StdCopy is a modified version of io.Copy. +// +// StdCopy copies from src to dstout or dsterr until either EOF is reached +// on src or an error occurs. It returns the number of bytes +// copied and the first error encountered while copying, if any. +// +// A successful Copy returns err == nil, not err == EOF. +// Because Copy is defined to read from src until EOF, it does +// not treat an EOF from Read as an error to be reported. +// +// The source needs to be writter via StdWriter, dstout or dsterr is selected +// based on the prefix added by StdWriter +func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { + var ( + buf = make([]byte, 32*1024+StdWriterPrefixLen+1) + bufLen = len(buf) + nr, nw int + er, ew error + out io.Writer + byteOrder binary.ByteOrder + frameSize int + ) + + // Check the machine's endianness + if CheckBigEndian() { + byteOrder = binary.BigEndian + } else { + byteOrder = binary.LittleEndian + } + + for { + // Make sure we have at least a full header + for nr < StdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + if er == io.EOF { + return written, nil + } + if er != nil { + return 0, er + } + nr += nr2 + } + + // Check the first byte to know where to write + switch buf[StdWriterFdIndex] { + case 0: + fallthrough + case 1: + // Write on stdout + out = dstout + case 2: + // Write on stderr + out = dsterr + default: + Debugf("Error selecting output fd: (%d)", buf[StdWriterFdIndex]) + return 0, ErrInvalidStdHeader + } + + // Retrieve the size of the frame + frameSize = int(byteOrder.Uint32(buf[StdWriterSizeIndex : StdWriterSizeIndex+4])) + + // Check if the buffer is big enough to read the frame. + // Extend it if necessary. + if frameSize+StdWriterPrefixLen > bufLen { + Debugf("Extending buffer cap.") + buf = append(buf, make([]byte, frameSize-len(buf)+1)...) + bufLen = len(buf) + } + + // While the amount of bytes read is less than the size of the frame + header, we keep reading + for nr < frameSize+StdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + if er == io.EOF { + return written, nil + } + if er != nil { + Debugf("Error reading frame: %s", er) + return 0, er + } + nr += nr2 + } + + // Write the retrieved frame (without header) + nw, ew = out.Write(buf[StdWriterPrefixLen : frameSize+StdWriterPrefixLen]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + Debugf("Error writing frame: %s", ew) + return 0, ew + } + // If the frame has not been fully written: error + if nw != frameSize { + Debugf("Error Short Write: (%d on %d)", nw, frameSize) + return 0, io.ErrShortWrite + } + + // Move the rest of the buffer to the beginning + copy(buf, buf[frameSize+StdWriterPrefixLen:]) + // Move the index + nr -= frameSize + StdWriterPrefixLen + } +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 18b695a3eb..d417690c0c 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -1021,90 +1021,3 @@ type StatusError struct { func (e *StatusError) Error() string { return fmt.Sprintf("Status: %d", e.Status) } - -type StdType []byte - -const StdWriterPrefixLen = 8 - -var ( - Stdin StdType = StdType("\001 stdin\002") - Stdout StdType = StdType("\001stdout\002") - Stderr StdType = StdType("\001stderr\002") -) - -type StdWriter struct { - io.Writer - prefix []byte -} - -func (w *StdWriter) Write(buf []byte) (n int, err error) { - n, err = w.Writer.Write(append(w.prefix, buf...)) - if n >= len(buf)+StdWriterPrefixLen { - n -= StdWriterPrefixLen - } - return n, err -} - -// NewStdWriter instanciate a new Writer based on the given type `t`. -// the utils package contains the valid parametres for `t`: -func NewStdWriter(w io.Writer, t StdType) *StdWriter { - if len(t) != StdWriterPrefixLen { - return nil - } - return &StdWriter{ - Writer: w, - prefix: []byte(t), - } -} - -// StdCopy is a modified version of io.Copy. -// -// StdCopy copies from src to dstout or dsterr until either EOF is reached -// on src or an error occurs. It returns the number of bytes -// copied and the first error encountered while copying, if any. -// -// A successful Copy returns err == nil, not err == EOF. -// Because Copy is defined to read from src until EOF, it does -// not treat an EOF from Read as an error to be reported. -// -// The source needs to be writter via StdWriter, dstout or dsterr is selected -// based on the prefix added by StdWriter -func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { - var ( - buf = make([]byte, 32*1024) - nw int - ew error - ) - - for { - nr, er := src.Read(buf) - if nr > 0 { - if bytes.Compare(buf[:StdWriterPrefixLen], Stdout) == 0 { - nw, ew = dstout.Write(buf[StdWriterPrefixLen:nr]) - } else if bytes.Compare(buf[:StdWriterPrefixLen], Stderr) == 0 { - nw, ew = dsterr.Write(buf[StdWriterPrefixLen:nr]) - } else if bytes.Compare(buf[:StdWriterPrefixLen], Stdin) == 0 { - nw, ew = dstout.Write(buf[StdWriterPrefixLen:nr]) - } - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr-StdWriterPrefixLen != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} From a21e18e65fae1c446b74667a0809729b27ce4687 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 11 Sep 2013 15:14:33 -0700 Subject: [PATCH 4/5] Add documentation Upstream-commit: 082d142024e69a817ed96772ee3057bbdeaa8d7d Component: engine --- components/engine/api.go | 12 +- .../docs/sources/api/attach_api_1.6.rst | 52 + .../docs/sources/api/docker_remote_api.rst | 19 +- .../sources/api/docker_remote_api_v1.6.rst | 1175 +++++++++++++++++ 4 files changed, 1251 insertions(+), 7 deletions(-) create mode 100644 components/engine/docs/sources/api/attach_api_1.6.rst create mode 100644 components/engine/docs/sources/api/docker_remote_api_v1.6.rst diff --git a/components/engine/api.go b/components/engine/api.go index 1b36221ba6..f82d609293 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -21,10 +21,12 @@ import ( "strings" ) -const APIVERSION = 1.5 -const DEFAULTHTTPHOST = "127.0.0.1" -const DEFAULTHTTPPORT = 4243 -const DEFAULTUNIXSOCKET = "/var/run/docker.sock" +const ( + APIVERSION = 1.6 + DEFAULTHTTPHOST = "127.0.0.1" + DEFAULTHTTPPORT = 4243 + DEFAULTUNIXSOCKET = "/var/run/docker.sock" +) type HttpApiFunc func(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error @@ -794,7 +796,7 @@ func postContainersAttach(srv *Server, version float64, w http.ResponseWriter, r fmt.Fprintf(outStream, "HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n") - if !c.Config.Tty && version >= 1.4 { + if !c.Config.Tty && version >= 1.6 { errStream = utils.NewStdWriter(outStream, utils.Stderr) outStream = utils.NewStdWriter(outStream, utils.Stdout) } else { diff --git a/components/engine/docs/sources/api/attach_api_1.6.rst b/components/engine/docs/sources/api/attach_api_1.6.rst new file mode 100644 index 0000000000..d165b12897 --- /dev/null +++ b/components/engine/docs/sources/api/attach_api_1.6.rst @@ -0,0 +1,52 @@ +:title: Attach stream API +:description: API Documentation for the Attach command in Docker +:keywords: API, Docker, Attach, Stream, REST, documentation + +================= +Docker Attach stream API +================= + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- This is the Attach stream API for Docker + +2. Format +========= + +The attach format is a Header and a Payload (frame). + +2.1 Header +^^^^^^^^^^ + +The header will contain the information on which stream write +the stream (stdout or stderr). +It also contain the size of the associated frame encoded on the last 4 bytes (uint32). + +It is encoded on the first 8 bytes like this: +header := [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4} + +STREAM_TYPE can be: +- 0: stdin (will be writen on stdout) +- 1: stdout +- 2: stderr + +SIZE1, SIZE2, SIZE3, SIZE4 are the 4 bytes of the uint32 size. + +2.1 Payload (frame) +^^^^^^^^^^^^^^^^^^^ + +The payload is the raw stream. + +3. Implementation +================= + +The simplest way to implement the Attach protocol is the following: + +1) Read 8 bytes +2) chose stdout or stderr depending on the first byte +3) Extract the frame size from the last 4 byets +4) Read the extracted size and output it on the correct output +5) Goto 1) diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 6712a4a57f..480e24bdaf 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -27,14 +27,29 @@ Docker Remote API 2. Versions =========== -The current version of the API is 1.5 +The current version of the API is 1.6 Calling /images//insert is the same as calling -/v1.5/images//insert +/v1.6/images//insert You can still call an old version of the api using /v1.0/images//insert +:doc:`docker_remote_api_v1.6` +***************************** + +What's new +---------- + +.. http:post:: /containers/(id)/attach + + **New!** You can now split stderr from stdout. This is done by prefixing + a header to each transmition. See :doc:`attach_api_1.6`. + The WebSocket attach is unchanged. + Note that attach calls on previous API version didn't change. Stdout and + stderr are merge. + + :doc:`docker_remote_api_v1.5` ***************************** diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.6.rst b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst new file mode 100644 index 0000000000..b98e770b46 --- /dev/null +++ b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst @@ -0,0 +1,1175 @@ +:title: Remote API v1.6 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +:orphan: + +====================== +Docker Remote API v1.6 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker daemon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":[{"PrivatePort": 2222, "PublicPort": 3333, "Type": "tcp"}], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":[], + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Privileged": false, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"", + "WorkingDir":"" + + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "", + "WorkingDir":"" + + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"], + "LxcConf":{"lxc.utsname":"docker"} + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a container +**************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ PREFIXED STREAM }} See :doc:`attach_api_1.6` + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Copy files or folders from a container +************************************** + +.. http:post:: /containers/(id)/copy + + Copy files or folders of container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/4fa6e0f0c678/copy HTTP/1.1 + Content-Type: application/json + + { + "Resource":"test.txt" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + When using this endpoint to pull an image from the registry, + the ``X-Registry-Auth`` header can be used to include a + base64-encoded AuthConfig object. + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in an image +************************* + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"", + "WorkingDir":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + The ``X-Registry-Auth`` header can be used to include a + base64-encoded AuthConfig object. + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success + :query q: suppress verbose build output + :query nocache: do not use the cache when building the image + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com", + "serveraddress":"https://index.docker.io/v1/" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false, + "IPv4Forwarding":true + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Monitor Docker's events +*********************** + +.. http:get:: /events + + Get events from docker, either in real time via streaming, or via polling (using `since`) + + **Example request**: + + .. sourcecode:: http + + POST /events?since=1374067924 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"create","id":"dfdf82bd3881","from":"base:latest","time":1374067924} + {"status":"start","id":"dfdf82bd3881","from":"base:latest","time":1374067924} + {"status":"stop","id":"dfdf82bd3881","from":"base:latest","time":1374067966} + {"status":"destroy","id":"dfdf82bd3881","from":"base:latest","time":1374067970} + + :query since: timestamp used for polling + :statuscode 200: no error + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + +.. code-block:: bash + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From c6a1c7cdae25522de5c9967ec9c9856ac536819c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 26 Sep 2013 15:59:02 -0700 Subject: [PATCH 5/5] Update docs + fix endian issue Upstream-commit: cb18a6e1b934b7f5c63f8931ee8fe51e2789c597 Component: engine --- components/engine/api_test.go | 4 +- .../docs/sources/api/attach_api_1.6.rst | 52 ------------------- .../docs/sources/api/docker_remote_api.rst | 6 +-- .../sources/api/docker_remote_api_v1.6.rst | 45 +++++++++++++++- components/engine/server.go | 10 ++-- components/engine/utils/stdcopy.go | 41 +++------------ 6 files changed, 63 insertions(+), 95 deletions(-) delete mode 100644 components/engine/docs/sources/api/attach_api_1.6.rst diff --git a/components/engine/api_test.go b/components/engine/api_test.go index f652b07844..034008b2eb 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -951,7 +951,7 @@ func TestPostContainersAttach(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string([]byte{1, 0, 0, 0, 6, 0, 0, 0})+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{1, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) } }) @@ -1040,7 +1040,7 @@ func TestPostContainersAttachStderr(t *testing.T) { }) setTimeout(t, "read/write assertion timed out", 2*time.Second, func() { - if err := assertPipe("hello\n", string([]byte{2, 0, 0, 0, 6, 0, 0, 0})+"hello", stdout, stdinPipe, 15); err != nil { + if err := assertPipe("hello\n", string([]byte{2, 0, 0, 0, 0, 0, 0, 6})+"hello", stdout, stdinPipe, 15); err != nil { t.Fatal(err) } }) diff --git a/components/engine/docs/sources/api/attach_api_1.6.rst b/components/engine/docs/sources/api/attach_api_1.6.rst deleted file mode 100644 index d165b12897..0000000000 --- a/components/engine/docs/sources/api/attach_api_1.6.rst +++ /dev/null @@ -1,52 +0,0 @@ -:title: Attach stream API -:description: API Documentation for the Attach command in Docker -:keywords: API, Docker, Attach, Stream, REST, documentation - -================= -Docker Attach stream API -================= - -.. contents:: Table of Contents - -1. Brief introduction -===================== - -- This is the Attach stream API for Docker - -2. Format -========= - -The attach format is a Header and a Payload (frame). - -2.1 Header -^^^^^^^^^^ - -The header will contain the information on which stream write -the stream (stdout or stderr). -It also contain the size of the associated frame encoded on the last 4 bytes (uint32). - -It is encoded on the first 8 bytes like this: -header := [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4} - -STREAM_TYPE can be: -- 0: stdin (will be writen on stdout) -- 1: stdout -- 2: stderr - -SIZE1, SIZE2, SIZE3, SIZE4 are the 4 bytes of the uint32 size. - -2.1 Payload (frame) -^^^^^^^^^^^^^^^^^^^ - -The payload is the raw stream. - -3. Implementation -================= - -The simplest way to implement the Attach protocol is the following: - -1) Read 8 bytes -2) chose stdout or stderr depending on the first byte -3) Extract the frame size from the last 4 byets -4) Read the extracted size and output it on the correct output -5) Goto 1) diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 480e24bdaf..9a6b72fafa 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -44,10 +44,10 @@ What's new .. http:post:: /containers/(id)/attach **New!** You can now split stderr from stdout. This is done by prefixing - a header to each transmition. See :doc:`attach_api_1.6`. + a header to each transmition. See :http:post:`/containers/(id)/attach`. The WebSocket attach is unchanged. - Note that attach calls on previous API version didn't change. Stdout and - stderr are merge. + Note that attach calls on the previous API version didn't change. Stdout and + stderr are merged. :doc:`docker_remote_api_v1.5` diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.6.rst b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst index b98e770b46..d84d5e85d0 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.6.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst @@ -468,7 +468,7 @@ Attach to a container HTTP/1.1 200 OK Content-Type: application/vnd.docker.raw-stream - {{ PREFIXED STREAM }} See :doc:`attach_api_1.6` + {{ STREAM }} :query logs: 1/True/true or 0/False/false, return logs. Default false :query stream: 1/True/true or 0/False/false, return stream. Default false @@ -480,6 +480,49 @@ Attach to a container :statuscode 404: no such container :statuscode 500: server error + **Stream details**: + + When using the TTY setting is enabled in + :http:post:`/containers/create`, the stream is the raw data + from the process PTY and client's stdin. When the TTY is + disabled, then the stream is multiplexed to separate stdout + and stderr. + + The format is a **Header** and a **Payload** (frame). + + **HEADER** + + The header will contain the information on which stream write + the stream (stdout or stderr). It also contain the size of + the associated frame encoded on the last 4 bytes (uint32). + + It is encoded on the first 8 bytes like this:: + + header := [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4} + + ``STREAM_TYPE`` can be: + + - 0: stdin (will be writen on stdout) + - 1: stdout + - 2: stderr + + ``SIZE1, SIZE2, SIZE3, SIZE4`` are the 4 bytes of the uint32 size encoded as big endian. + + **PAYLOAD** + + The payload is the raw stream. + + **IMPLEMENTATION** + + The simplest way to implement the Attach protocol is the following: + + 1) Read 8 bytes + 2) chose stdout or stderr depending on the first byte + 3) Extract the frame size from the last 4 byets + 4) Read the extracted size and output it on the correct output + 5) Goto 1) + + Wait a container **************** diff --git a/components/engine/server.go b/components/engine/server.go index b40f60556e..38b7bcf147 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1208,16 +1208,20 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std } else { dec := json.NewDecoder(cLog) for { - var l utils.JSONLog - if err := dec.Decode(&l); err == io.EOF { + l := &utils.JSONLog{} + + if err := dec.Decode(l); err == io.EOF { break } else if err != nil { utils.Debugf("Error streaming logs: %s", err) break } - if (l.Stream == "stdout" && stdout) || (l.Stream == "stderr" && stderr) { + if l.Stream == "stdout" && stdout { fmt.Fprintf(outStream, "%s", l.Log) } + if l.Stream == "stderr" && stderr { + fmt.Fprintf(errStream, "%s", l.Log) + } } } } diff --git a/components/engine/utils/stdcopy.go b/components/engine/utils/stdcopy.go index 4965ea41b0..42dad738e4 100644 --- a/components/engine/utils/stdcopy.go +++ b/components/engine/utils/stdcopy.go @@ -4,18 +4,8 @@ import ( "encoding/binary" "errors" "io" - "unsafe" ) -func CheckBigEndian() bool { - var x uint32 = 0x01020304 - - if *(*byte)(unsafe.Pointer(&x)) == 0x01 { - return true - } - return false -} - const ( StdWriterPrefixLen = 8 StdWriterFdIndex = 0 @@ -32,16 +22,15 @@ var ( type StdWriter struct { io.Writer - prefix StdType - sizeBuf []byte - byteOrder binary.ByteOrder + prefix StdType + sizeBuf []byte } func (w *StdWriter) Write(buf []byte) (n int, err error) { if w == nil || w.Writer == nil { return 0, errors.New("Writer not instanciated") } - w.byteOrder.PutUint32(w.prefix[4:], uint32(len(buf))) + binary.BigEndian.PutUint32(w.prefix[4:], uint32(len(buf))) buf = append(w.prefix[:], buf...) n, err = w.Writer.Write(buf) @@ -55,18 +44,10 @@ func NewStdWriter(w io.Writer, t StdType) *StdWriter { return nil } - var bo binary.ByteOrder - - if CheckBigEndian() { - bo = binary.BigEndian - } else { - bo = binary.LittleEndian - } return &StdWriter{ - Writer: w, - prefix: t, - sizeBuf: make([]byte, 4), - byteOrder: bo, + Writer: w, + prefix: t, + sizeBuf: make([]byte, 4), } } @@ -91,17 +72,9 @@ func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) nr, nw int er, ew error out io.Writer - byteOrder binary.ByteOrder frameSize int ) - // Check the machine's endianness - if CheckBigEndian() { - byteOrder = binary.BigEndian - } else { - byteOrder = binary.LittleEndian - } - for { // Make sure we have at least a full header for nr < StdWriterPrefixLen { @@ -132,7 +105,7 @@ func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) } // Retrieve the size of the frame - frameSize = int(byteOrder.Uint32(buf[StdWriterSizeIndex : StdWriterSizeIndex+4])) + frameSize = int(binary.BigEndian.Uint32(buf[StdWriterSizeIndex : StdWriterSizeIndex+4])) // Check if the buffer is big enough to read the frame. // Extend it if necessary.