From 2394c9c8dca18831eea5f2b3d67da9ea3c5d18f2 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 3 Jul 2013 17:11:00 +0000 Subject: [PATCH 01/54] add last version Upstream-commit: 64450ae3f89b8f9b5288224c5a7d109a166cf22a Component: engine --- components/engine/commands.go | 9 +++++++++ components/engine/utils/utils.go | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/components/engine/commands.go b/components/engine/commands.go index 6e1e5e88c2..e275bdd671 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -407,6 +407,15 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if out.GoVersion != "" { fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } + + release := utils.GetReleaseVersion() + if release != "" { + fmt.Fprintf(cli.out, "Last stable version: %s", release) + if VERSION != release || out.Version != release { + fmt.Fprintf(cli.out, ", please update docker") + } + fmt.Fprintf(cli.out, "\n") + } return nil } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 2f2a52867e..b4a41ea420 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -687,3 +687,25 @@ func ParseHost(host string, port int, addr string) string { } return fmt.Sprintf("tcp://%s:%d", host, port) } + +func GetReleaseVersion() string { + type githubTag struct { + Name string `json:"name"` + } + + resp, err := http.Get("https://api.github.com/repos/dotcloud/docker/tags") + if err != nil { + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + var tags []githubTag + err = json.Unmarshal(body, &tags) + if err != nil || len(tags) == 0 { + return "" + } + return strings.TrimPrefix(tags[0].Name, "v") +} From 4b7018962aac5e057cc6a5c674b6871dca26e954 Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:36:05 +0100 Subject: [PATCH 02/54] docker.upstart: avoid spawning a `sh` process start script / end script create an intermediate sh process. Upstream-commit: 24dd50490a027f01ea086eb90663d53348fa770e Component: engine --- components/engine/packaging/ubuntu/docker.upstart | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 2bd5565ee7..1d35d7a493 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -4,6 +4,4 @@ start on runlevel [2345] stop on starting rc RUNLEVEL=[016] respawn -script - /usr/bin/docker -d -end script +exec /usr/bin/docker -d From 632014a7bf21f6db9ed19062cb00f454e5359ac9 Mon Sep 17 00:00:00 2001 From: Jonas Pfenniger Date: Mon, 15 Jul 2013 11:41:19 +0100 Subject: [PATCH 03/54] docker.upstart: use the same start/stop events as sshd Is probably more solid Upstream-commit: 0900d3b7a6e94bfa42e3d4ac6dc6f5542f65a9b0 Component: engine --- components/engine/packaging/ubuntu/docker.upstart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/packaging/ubuntu/docker.upstart b/components/engine/packaging/ubuntu/docker.upstart index 1d35d7a493..f4d2fbe922 100644 --- a/components/engine/packaging/ubuntu/docker.upstart +++ b/components/engine/packaging/ubuntu/docker.upstart @@ -1,7 +1,8 @@ description "Run docker" -start on runlevel [2345] -stop on starting rc RUNLEVEL=[016] +start on filesystem or runlevel [2345] +stop on runlevel [!2345] + respawn exec /usr/bin/docker -d From 9c8c35f4bd8616581c8840eb4b72b08630d89ab5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 18 Jul 2013 20:56:41 +0000 Subject: [PATCH 04/54] handle -dev Upstream-commit: 0afed3eded0950b5899729714fb11d6a7322301e Component: engine --- components/engine/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 5f3913f9c2..7b70faa198 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -436,7 +436,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { release := utils.GetReleaseVersion() if release != "" { fmt.Fprintf(cli.out, "Last stable version: %s", release) - if VERSION != release || out.Version != release { + if strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release { fmt.Fprintf(cli.out, ", please update docker") } fmt.Fprintf(cli.out, "\n") From 37e3d486fa87ae31293b385a6f957865d61b13bb Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:06:32 +0000 Subject: [PATCH 05/54] add ps args to docker top Upstream-commit: cfec1c3e1b88ceeca73144198df7a210ed3dc421 Component: engine --- components/engine/api.go | 6 +++++- components/engine/api_params.go | 6 ++---- components/engine/api_test.go | 27 ++++++++++++++++++--------- components/engine/commands.go | 17 +++++++++++------ components/engine/server.go | 29 ++++++++++++++--------------- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index b6ab7badfa..d3b84df5f9 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -255,8 +255,12 @@ func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *ht if vars == nil { return fmt.Errorf("Missing parameter") } + if err := parseForm(r); err != nil { + return err + } name := vars["name"] - procsStr, err := srv.ContainerTop(name) + ps_args := r.Form.Get("ps_args") + procsStr, err := srv.ContainerTop(name, ps_args) if err != nil { return err } diff --git a/components/engine/api_params.go b/components/engine/api_params.go index b371ca314f..2296ee792e 100644 --- a/components/engine/api_params.go +++ b/components/engine/api_params.go @@ -27,10 +27,8 @@ type APIInfo struct { } type APITop struct { - PID string - Tty string - Time string - Cmd string + Titles []string + Processes [][]string } type APIRmi struct { diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 17ada96eab..9b7f08d1db 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -444,24 +444,33 @@ func TestGetContainersTop(t *testing.T) { } r := httptest.NewRecorder() - if err := getContainersTop(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil { + req, err := http.NewRequest("GET", "/"+container.ID+"/top?ps_args=u", bytes.NewReader([]byte{})) + if err != nil { t.Fatal(err) } - procs := []APITop{} + if err := getContainersTop(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + procs := APITop{} if err := json.Unmarshal(r.Body.Bytes(), &procs); err != nil { t.Fatal(err) } - if len(procs) != 2 { - t.Fatalf("Expected 2 processes, found %d.", len(procs)) + if len(procs.Titles) != 11 { + t.Fatalf("Expected 11 titles, found %d.", len(procs.Titles)) + } + if procs.Titles[0] != "USER" || procs.Titles[10] != "COMMAND" { + t.Fatalf("Expected Titles[0] to be USER and Titles[10] to be COMMAND, found %s and %s.", procs.Titles[0], procs.Titles[10]) } - if procs[0].Cmd != "sh" && procs[0].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[0].Cmd) + if len(procs.Processes) != 2 { + t.Fatalf("Expected 2 processes, found %d.", len(procs.Processes)) } - - if procs[1].Cmd != "sh" && procs[1].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[1].Cmd) + if procs.Processes[0][10] != "/bin/sh" && procs.Processes[0][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[0][10]) + } + if procs.Processes[1][10] != "/bin/sh" && procs.Processes[1][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[1][10]) } } diff --git a/components/engine/commands.go b/components/engine/commands.go index 936b23fea2..b25e928efa 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -585,23 +585,28 @@ func (cli *DockerCli) CmdTop(args ...string) error { if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() != 1 { + if cmd.NArg() == 0 { cmd.Usage() return nil } - body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top", nil) + val := url.Values{} + if cmd.NArg() > 1 { + val.Set("ps_args", strings.Join(cmd.Args()[1:], " ")) + } + + body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top?"+val.Encode(), nil) if err != nil { return err } - var procs []APITop + procs := APITop{} err = json.Unmarshal(body, &procs) if err != nil { return err } w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) - fmt.Fprintln(w, "PID\tTTY\tTIME\tCMD") - for _, proc := range procs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", proc.PID, proc.Tty, proc.Time, proc.Cmd) + fmt.Fprintln(w, strings.Join(procs.Titles, "\t")) + for _, proc := range procs.Processes { + fmt.Fprintln(w, strings.Join(proc, "\t")) } w.Flush() return nil diff --git a/components/engine/server.go b/components/engine/server.go index 954bbb208f..ae5f605267 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -249,35 +249,34 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { } -func (srv *Server) ContainerTop(name string) ([]APITop, error) { +func (srv *Server) ContainerTop(name, ps_args string) (*APITop, error) { if container := srv.runtime.Get(name); container != nil { - output, err := exec.Command("lxc-ps", "--name", container.ID).CombinedOutput() + output, err := exec.Command("lxc-ps", "--name", container.ID, "--", ps_args).CombinedOutput() if err != nil { return nil, fmt.Errorf("Error trying to use lxc-ps: %s (%s)", err, output) } - var procs []APITop + procs := APITop{} for i, line := range strings.Split(string(output), "\n") { - if i == 0 || len(line) == 0 { + if len(line) == 0 { continue } - proc := APITop{} + words := []string{} scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) if !scanner.Scan() { return nil, fmt.Errorf("Error trying to use lxc-ps") } // no scanner.Text because we skip container id - scanner.Scan() - proc.PID = scanner.Text() - scanner.Scan() - proc.Tty = scanner.Text() - scanner.Scan() - proc.Time = scanner.Text() - scanner.Scan() - proc.Cmd = scanner.Text() - procs = append(procs, proc) + for scanner.Scan() { + words = append(words, scanner.Text()) + } + if i == 0 { + procs.Titles = words + } else { + procs.Processes = append(procs.Processes, words) + } } - return procs, nil + return &procs, nil } return nil, fmt.Errorf("No such container: %s", name) From 797b7b513bdcdbda74626e391699d652d7106f8c Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 19 Jul 2013 10:34:55 +0000 Subject: [PATCH 06/54] bump api version to 1.4 Upstream-commit: eb4a0271fbb7318a7d66911ccd388c048db0c2fa Component: engine --- components/engine/api.go | 5 +- .../docs/sources/api/docker_remote_api.rst | 17 +- .../sources/api/docker_remote_api_v1.4.rst | 1093 +++++++++++++++++ 3 files changed, 1112 insertions(+), 3 deletions(-) create mode 100644 components/engine/docs/sources/api/docker_remote_api_v1.4.rst diff --git a/components/engine/api.go b/components/engine/api.go index d3b84df5f9..a3dd52108e 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -17,7 +17,7 @@ import ( "strings" ) -const APIVERSION = 1.3 +const APIVERSION = 1.4 const DEFAULTHTTPHOST string = "127.0.0.1" const DEFAULTHTTPPORT int = 4243 @@ -252,6 +252,9 @@ func getContainersChanges(srv *Server, version float64, w http.ResponseWriter, r } func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if version < 1.4 { + return fmt.Errorf("top was improved a lot since 1.3, Please upgrade your docker client.") + } if vars == nil { return fmt.Errorf("Missing parameter") } diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 183347c23b..a08fb46940 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -19,8 +19,8 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 -Calling /images//insert is the same as calling /v1.3/images//insert +The current verson of the API is 1.4 +Calling /images//insert is the same as calling /v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert :doc:`docker_remote_api_v1.3` @@ -31,6 +31,18 @@ What's new Listing processes (/top): +- You can now use ps args with docker top, like `docker top aux` + +:doc:`docker_remote_api_v1.3` +***************************** + +docker v0.5.0 51f6c4a_ + +What's new +---------- + +Listing processes (/top): + - List the processes inside a container @@ -109,6 +121,7 @@ Initial version .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4 .. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168 +.. _51f6c4a: https://github.com/dotcloud/docker/commit/51f6c4a7372450d164c61e0054daf0223ddbd909 ================================== Docker Remote API Client Libraries diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst new file mode 100644 index 0000000000..c42adb286f --- /dev/null +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -0,0 +1,1093 @@ +:title: Remote API v1.3 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +====================== +Docker Remote API v1.3 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker deamon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"] + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: tag to be applied to the resulting image in case of success + :query q: suppress verbose build output + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + + docker -d -H="192.168.1.9:4243" -api-enable-cors + From ff1a26c4e5d8f0fb8115a6f03e3b9176f45a0cfd Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Tue, 23 Jul 2013 13:05:06 -0700 Subject: [PATCH 07/54] Docs: Fixed navigaton links to about page and community page Website: Removed the website sources from the repo. The website sources are now hosted on github.com/dotcloud/www.docker.io/ Upstream-commit: e701dce33978a0206627a02b258da6a0269c4b60 Component: engine --- components/engine/docs/theme/MAINTAINERS | 2 +- .../engine/docs/theme/docker/layout.html | 8 +- components/engine/docs/website/MAINTAINERS | 1 - components/engine/docs/website/dotcloud.yml | 2 - .../docs/website/gettingstarted/index.html | 220 ----------- components/engine/docs/website/index.html | 359 ------------------ components/engine/docs/website/nginx.conf | 6 - components/engine/docs/website/static | 1 - 8 files changed, 5 insertions(+), 594 deletions(-) delete mode 100644 components/engine/docs/website/MAINTAINERS delete mode 100644 components/engine/docs/website/dotcloud.yml delete mode 100644 components/engine/docs/website/gettingstarted/index.html delete mode 100644 components/engine/docs/website/index.html delete mode 100644 components/engine/docs/website/nginx.conf delete mode 120000 components/engine/docs/website/static diff --git a/components/engine/docs/theme/MAINTAINERS b/components/engine/docs/theme/MAINTAINERS index 6df367c073..606a1dd746 100644 --- a/components/engine/docs/theme/MAINTAINERS +++ b/components/engine/docs/theme/MAINTAINERS @@ -1 +1 @@ -Thatcher Penskens +Thatcher Peskens diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 198cd5d7d8..0b22f22fab 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -68,12 +68,12 @@
diff --git a/components/engine/docs/website/MAINTAINERS b/components/engine/docs/website/MAINTAINERS deleted file mode 100644 index 6df367c073..0000000000 --- a/components/engine/docs/website/MAINTAINERS +++ /dev/null @@ -1 +0,0 @@ -Thatcher Penskens diff --git a/components/engine/docs/website/dotcloud.yml b/components/engine/docs/website/dotcloud.yml deleted file mode 100644 index 5a8f50f9e9..0000000000 --- a/components/engine/docs/website/dotcloud.yml +++ /dev/null @@ -1,2 +0,0 @@ -www: - type: static \ No newline at end of file diff --git a/components/engine/docs/website/gettingstarted/index.html b/components/engine/docs/website/gettingstarted/index.html deleted file mode 100644 index de0cc3512d..0000000000 --- a/components/engine/docs/website/gettingstarted/index.html +++ /dev/null @@ -1,220 +0,0 @@ - - - - - - - - - - Docker - the Linux container runtime - - - - - - - - - - - - - - - - - - - - -
-
-
- -
- -
- -
- -
-
-
-
- - -
-
- -
- - - -

GETTING STARTED

-
- -
- -
- -
-
- Docker is still under heavy development. It should not yet be used in production. Check the repo for recent progress. -
-
-
-
-

- - Installing on Ubuntu

- -

Requirements

-
    -
  • Ubuntu 12.04 (LTS) (64-bit)
    • -
  • or Ubuntu 12.10 (quantal) (64-bit)
    • -
  • The 3.8 Linux Kernel
    • -
-
    -
  1. -

    Install dependencies

    - The linux-image-extra package is only needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - 
    sudo apt-get install linux-image-extra-`uname -r`
    - - -
    2. -
  2. -

    Install Docker

    -

    Add the Ubuntu PPA (Personal Package Archive) sources to your apt sources list, update and install.

    -

    This may import a new GPG key (key 63561DC6: public key "Launchpad PPA for dotcloud team" imported).

    -
    - 
    sudo apt-get install software-properties-common
    - 
    sudo add-apt-repository ppa:dotcloud/lxc-docker
    - 
    sudo apt-get update
    - 
    sudo apt-get install lxc-docker
    -
    - - -
    3. - -
  3. -

    Run!

    - -
    - 
    docker run -i -t ubuntu /bin/bash
    -
    -
    4. - Continue with the Hello world example.
    - Or check more detailed installation instructions -
-
- -
-

Contributing to Docker

- -

Want to hack on Docker? Awesome! We have some instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete.

-
- -
-
-
-

Quick install on other operating systems

-

For other operating systems we recommend and provide a streamlined install with virtualbox, - vagrant and an Ubuntu virtual machine.

- - - -
- -
-

Questions? Want to get in touch?

-

There are several ways to get in touch:

-

Join the discussion on IRC. We can be found in the #docker channel on chat.freenode.net

-

Discussions happen on our google group: docker-club at googlegroups.com

-

All our development and decisions are made out in the open on Github github.com/dotcloud/docker

-

Get help on using Docker by asking on Stackoverflow

-

And of course, tweet your tweets to twitter.com/getdocker

-
- - -
-
- Fill out my online form. -
- -
- -
-
-
- - -
- -
- - - - - - - - - - - diff --git a/components/engine/docs/website/index.html b/components/engine/docs/website/index.html deleted file mode 100644 index f6f4efbccb..0000000000 --- a/components/engine/docs/website/index.html +++ /dev/null @@ -1,359 +0,0 @@ - - - - - - - - - - - Docker - the Linux container engine - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - - -
- -
-
-
-
- - -
-
- -
-
- -
-
- docker letters - -

The Linux container engine

-
- -
- -
- Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers which are independent of hardware, language, framework, packaging system and hosting provider. -
- -
- - - - -
- -
-
- -
-
- -
-
-
-
-
- -
-
- -
-
-

Heterogeneous payloads

-

Any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all.

-

Any server

-

Docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments.

-

Isolation

-

Docker isolates processes from each other and from the underlying host, using lightweight containers.

-

Repeatability

-

Because each container is isolated in its own filesystem, they behave the same regardless of where, when, and alongside what they run.

-
-
-
-
- we're hiring -
-
-

Do you think it is cool to hack on docker? Join us!

-
    -
  • Work on open source
    • -
  • Program in Go
    • -
- read more -
-
- -
-
-
-
-

New! Docker Index

- On the Docker Index you can find and explore pre-made container images. It allows you to share your images and download them. - -

- -
- DOCKER index -
- -   - - -
-
-
- Fill out my online form. -
- -
-
-
- -
- -
- -
-
-
- - Mitchell Hashimoto ‏@mitchellh: Docker launched today. It is incredible. They’re also working RIGHT NOW on a Vagrant provider. LXC is COMING!! -
-
-
-
- - Adam Jacob ‏@adamhjk: Docker is clearly the right idea. @solomonstre absolutely killed it. Containerized app deployment is the future, I think. -
-
-
-
-
-
- - Matt Townsend ‏@mtownsend: I have a serious code crush on docker.io - it's Lego for PaaS. Motherfucking awesome Lego. -
-
-
-
- - Rob Harrop ‏@robertharrop: Impressed by @getdocker - it's all kinds of magic. Serious rethink of AWS architecture happening @skillsmatter. -
-
-
-
-
-
- - John Willis @botchagalupe: IMHO docker is to paas what chef was to Iaas 4 years ago -
-
-
-
- - John Feminella ‏@superninjarobot: So, @getdocker is pure excellence. If you've ever wished for arbitrary, PaaS-agnostic, lxc/aufs Linux containers, this is your jam! -
-
-
-
-
-
- - David Romulan ‏@destructuring: I haven't had this much fun since AWS -
-
-
-
- - Ricardo Gladwell ‏@rgladwell: wow @getdocker is either amazing or totally stupid -
-
- -
-
- -
-
-
- -
- -

Notable features

- -
    -
  • Filesystem isolation: each process container runs in a completely separate root filesystem.
    • -
  • Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups.
    • -
  • Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own.
    • -
  • Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremely fast, memory-cheap and disk-cheap.
    • -
  • Logging: the standard streams (stdout/stderr/stdin) of each process container is collected and logged for real-time or batch retrieval.
    • -
  • Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required.
    • -
  • Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell.
    • -
- -

Under the hood

- -

Under the hood, Docker is built on the following components:

- -
    -
  • The cgroup and namespacing capabilities of the Linux kernel;
    • -
  • AUFS, a powerful union filesystem with copy-on-write capabilities;
    • -
  • The Go programming language;
    • -
  • lxc, a set of convenience scripts to simplify the creation of linux containers.
    • -
- -

Who started it

-

- Docker is an open-source implementation of the deployment engine which powers dotCloud, a popular Platform-as-a-Service.

- -

It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands - of applications and databases. -

- -
-
- -
- - -
-

Twitter

- - -
- -
-
- -
- - -
- -
- - - - - - - - - - - - diff --git a/components/engine/docs/website/nginx.conf b/components/engine/docs/website/nginx.conf deleted file mode 100644 index 97ffd2c0e5..0000000000 --- a/components/engine/docs/website/nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ - -# rule to redirect original links created when hosted on github pages -rewrite ^/documentation/(.*).html http://docs.docker.io/en/latest/$1/ permanent; - -# rewrite the stuff which was on the current page -rewrite ^/gettingstarted.html$ /gettingstarted/ permanent; diff --git a/components/engine/docs/website/static b/components/engine/docs/website/static deleted file mode 120000 index 95bc97aa10..0000000000 --- a/components/engine/docs/website/static +++ /dev/null @@ -1 +0,0 @@ -../theme/docker/static \ No newline at end of file From 2cfc902d2966e236fe2c32855b6bbbe49de2b343 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:18:34 +0000 Subject: [PATCH 08/54] ensure mount in commit Upstream-commit: f385f1860bdebe2434bb122bd5ac8fec85687970 Component: engine --- components/engine/builder.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/engine/builder.go b/components/engine/builder.go index ab07233cb8..420370b1e6 100644 --- a/components/engine/builder.go +++ b/components/engine/builder.go @@ -124,6 +124,10 @@ func (builder *Builder) Create(config *Config) (*Container, error) { func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. + if err := container.EnsureMounted(); err != nil { + return nil, err + } + rwTar, err := container.ExportRw() if err != nil { return nil, err From 85f64e1df75a5e81b45ae149e44e30dc6d015b65 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 25 Jul 2013 15:20:56 +0000 Subject: [PATCH 09/54] add regression test + go fmt Upstream-commit: 48833c7b0784bc7055f29dd3448df3af93c462ec Component: engine --- components/engine/server_test.go | 21 +++++++++++++++++++++ components/engine/utils/utils.go | 7 +++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..f67b8fb57c 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -90,6 +90,27 @@ func TestCreateRm(t *testing.T) { } +func TestCommit(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) + if err != nil { + t.Fatal(err) + } + + id, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { + t.Fatal(err) + } +} + func TestCreateStartRestartStopStartKillRm(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 795426f1f2b90b3ddabcbe8be4af37ee44737831 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 23 Jul 2013 11:37:13 -0700 Subject: [PATCH 10/54] Make sure the cookie is used in all registry queries Upstream-commit: 3425c1b84c3f58ac5bb2feb91c4901b61561c58c Component: engine --- components/engine/registry/registry.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index e6f4f592e2..adef1c7baa 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -109,7 +109,14 @@ func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } - return c.Do(req) + res, err := c.Do(req) + if err != nil { + return nil, err + } + if len(res.Cookies()) > 0 { + c.Jar.SetCookies(req.URL, res.Cookies()) + } + return res, err } // Set the user agent field in the header based on the versions provided @@ -135,7 +142,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) @@ -182,7 +189,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -210,7 +217,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) ( } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -231,7 +238,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -326,7 +333,7 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } @@ -341,9 +348,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() - if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) - } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { From 7c51899dd52c23c8ae59b5660314c2d0df5c5e7a Mon Sep 17 00:00:00 2001 From: Fareed Dudhia Date: Tue, 9 Jul 2013 07:01:45 +0000 Subject: [PATCH 11/54] Fixes 1136; Reopened from 1175 with latest changes. Upstream-commit: d86898b0142f2f9a834aa0c727b10d62ef647262 Component: engine --- components/engine/buildfile.go | 69 +++++++++++++++++++++++++---- components/engine/buildfile_test.go | 42 +++++++++++++++++- components/engine/utils/utils.go | 7 ++- 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 75ebdd7a7c..c5171aaa91 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -11,6 +11,7 @@ import ( "os" "path" "reflect" + "regexp" "strings" ) @@ -67,6 +68,9 @@ func (b *buildFile) CmdFrom(name string) error { } b.image = image.ID b.config = &Config{} + if b.config.Env == nil || len(b.config.Env) == 0 { + b.config.Env = append(b.config.Env, "HOME=/", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + } return nil } @@ -112,6 +116,40 @@ func (b *buildFile) CmdRun(args string) error { return nil } +func (b *buildFile) FindEnvKey(key string) int { + for k, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + if key == envParts[0] { + return k + } + } + return -1 +} + +func (b *buildFile) ReplaceEnvMatches(value string) (string, error) { + exp, err := regexp.Compile("(\\\\\\\\+|[^\\\\]|\\b|\\A)\\$({?)([[:alnum:]_]+)(}?)") + if err != nil { + return value, err + } + matches := exp.FindAllString(value, -1) + for _, match := range matches { + match = match[strings.Index(match, "$"):] + matchKey := strings.Trim(match, "${}") + + for _, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + envKey := envParts[0] + envValue := envParts[1] + + if envKey == matchKey { + value = strings.Replace(value, match, envValue, -1) + break + } + } + } + return value, nil +} + func (b *buildFile) CmdEnv(args string) error { tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -120,14 +158,19 @@ func (b *buildFile) CmdEnv(args string) error { key := strings.Trim(tmp[0], " \t") value := strings.Trim(tmp[1], " \t") - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } + envKey := b.FindEnvKey(key) + replacedValue, err := b.ReplaceEnvMatches(value) + if err != nil { + return err } - b.config.Env = append(b.config.Env, key+"="+value) - return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s=%s", key, value)) + replacedVar := fmt.Sprintf("%s=%s", key, replacedValue) + + if envKey >= 0 { + b.config.Env[envKey] = replacedVar + return nil + } + b.config.Env = append(b.config.Env, replacedVar) + return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } func (b *buildFile) CmdCmd(args string) error { @@ -260,8 +303,16 @@ func (b *buildFile) CmdAdd(args string) error { if len(tmp) != 2 { return fmt.Errorf("Invalid ADD format") } - orig := strings.Trim(tmp[0], " \t") - dest := strings.Trim(tmp[1], " \t") + + orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t")) + if err != nil { + return err + } + + dest, err := b.ReplaceEnvMatches(strings.Trim(tmp[1], " \t")) + if err != nil { + return err + } cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index b7eca52336..78e53b8419 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -129,6 +129,38 @@ CMD Hello world nil, nil, }, + + { + ` +from {IMAGE} +env FOO /foo/baz +env BAR /bar +env BAZ $BAR +env FOOPATH $PATH:$FOO +run [ "$BAR" = "$BAZ" ] +run [ "$FOOPATH" = "$PATH:/foo/baz" ] +`, + nil, + nil, + }, + + { + ` +from {IMAGE} +env FOO /bar +env TEST testdir +env BAZ /foobar +add testfile $BAZ/ +add $TEST $FOO +run [ "$(cat /foobar/testfile)" = "test1" ] +run [ "$(cat /bar/withfile)" = "test2" ] +`, + [][2]string{ + {"testfile", "test1"}, + {"testdir/withfile", "test2"}, + }, + nil, + }, } // FIXME: test building with 2 successive overlapping ADD commands @@ -242,8 +274,14 @@ func TestBuildEnv(t *testing.T) { env port 4243 `, nil, nil}, t) - - if img.Config.Env[0] != "port=4243" { + hasEnv := false + for _, envVar := range img.Config.Env { + if envVar == "port=4243" { + hasEnv = true + break + } + } + if !hasEnv { t.Fail() } } diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool From 4118c3f05998e5880d5d5d1e9a878e3400e96760 Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 01:10:42 -0700 Subject: [PATCH 12/54] Change reserve-compatibility to reverse-compatibility Upstream-commit: 4ebe2cf348915415c34503aca7a5663177e0002f Component: engine --- components/engine/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/container.go b/components/engine/container.go index ec4abffc1b..d0b6ca4ce2 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -52,7 +52,7 @@ type Container struct { waitLock chan struct{} Volumes map[string]string - // Store rw/ro in a separate structure to preserve reserve-compatibility on-disk. + // Store rw/ro in a separate structure to preserve reverse-compatibility on-disk. // Easier than migrating older container configs :) VolumesRW map[string]bool } From 8f7401af36bf513edf440d779a1be4e7f7177b44 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 09:19:26 +0000 Subject: [PATCH 13/54] fix wrong untag when using rmi via id Upstream-commit: e608296bc62ceeaf41ebf2bc80b21c0a1883d4f0 Component: engine --- components/engine/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..fa8d8a0262 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -995,6 +995,9 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo + if len(strings.Split(repoAndTag, ":")) > 1 { + tag = strings.Split(repoAndTag, ":")[1] + } } else if repoName != parsedRepo { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict From 71ca6b437812a8ce19b134015e1e9781ab62e16d Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:04:46 +0000 Subject: [PATCH 14/54] fix docs Upstream-commit: 513a5674831fd952fc8e7b8fbdbc4939393ecb5b Component: engine --- components/engine/docs/sources/api/docker_remote_api.rst | 6 +++--- .../engine/docs/sources/api/docker_remote_api_v1.4.rst | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index d07c7634b9..0a1ff21cf1 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -26,15 +26,15 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 +The current verson of the API is 1.4 Calling /images//insert is the same as calling -/v1.3/images//insert +/v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert -:doc:`docker_remote_api_v1.3` +:doc:`docker_remote_api_v1.4` ***************************** What's new diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index c42adb286f..6ee0b35fa2 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -1,9 +1,9 @@ -:title: Remote API v1.3 +:title: Remote API v1.4 :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation ====================== -Docker Remote API v1.3 +Docker Remote API v1.4 ====================== .. contents:: Table of Contents From cdbe046a767d82cc86c585545ff89f52fd3f59f0 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Fri, 26 Jul 2013 10:01:41 +0000 Subject: [PATCH 15/54] add regression test Upstream-commit: e592f1b298c778d0b9adfd6751f5fe1843a7001d Component: engine --- components/engine/server.go | 2 +- components/engine/server_test.go | 86 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/components/engine/server.go b/components/engine/server.go index fa8d8a0262..56f738a5cd 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -995,7 +995,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo - if len(strings.Split(repoAndTag, ":")) > 1 { + if len(srv.runtime.repositories.ByID()[img.ID]) == 1 && len(strings.Split(repoAndTag, ":")) > 1 { tag = strings.Split(repoAndTag, ":")[1] } } else if repoName != parsedRepo { diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..b6b21cc75f 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -2,6 +2,7 @@ package docker import ( "github.com/dotcloud/docker/utils" + "strings" "testing" "time" ) @@ -203,3 +204,88 @@ func TestLogEvent(t *testing.T) { }) } + +func TestRmi(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + srv := &Server{runtime: runtime} + + initialImages, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + if err != nil { + t.Fatal(err) + } + + containerID, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + err = srv.ContainerTag(imageID, "test", "0.1", false) + if err != nil { + t.Fatal(err) + } + + containerID, err = srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + _, err = srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + images, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 2 { + t.Fatalf("Expected 2 new images, found %d.", len(images)-len(initialImages)) + } + + _, err = srv.ImageDelete(imageID, true) + if err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 1 { + t.Fatalf("Expected 1 new image, found %d.", len(images)-len(initialImages)) + } + + for _, image := range images { + if strings.Contains(unitTestImageID, image.ID) { + continue + } + if image.Repository == "" { + t.Fatalf("Expected tagged image, got untagged one.") + } + } +} From d391806121d62d6538dab5ad1305a6169a11ed36 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 26 Jul 2013 10:21:17 -0700 Subject: [PATCH 16/54] Clean up 'manifesto' in docs Upstream-commit: a97d858b2a0d6bc4d044a241036b97fd78f93022 Component: engine --- .../docs/sources/concepts/manifesto.rst | 61 ------------------- 1 file changed, 61 deletions(-) diff --git a/components/engine/docs/sources/concepts/manifesto.rst b/components/engine/docs/sources/concepts/manifesto.rst index ae09647094..7dd4b4bdda 100644 --- a/components/engine/docs/sources/concepts/manifesto.rst +++ b/components/engine/docs/sources/concepts/manifesto.rst @@ -4,10 +4,6 @@ .. _dockermanifesto: -*(This was our original Welcome page, but it is a bit forward-looking -for docs, and maybe not enough vision for a true manifesto. We'll -reveal more vision in the future to make it more Manifesto-y.)* - Docker Manifesto ---------------- @@ -131,60 +127,3 @@ sitting 10 miles away. With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - -Standard Container Specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(TODO) - -Image format -~~~~~~~~~~~~ - -Standard operations -~~~~~~~~~~~~~~~~~~~ - -- Copy -- Run -- Stop -- Wait -- Commit -- Attach standard streams -- List filesystem changes -- ... - -Execution environment -~~~~~~~~~~~~~~~~~~~~~ - -Root filesystem -^^^^^^^^^^^^^^^ - -Environment variables -^^^^^^^^^^^^^^^^^^^^^ - -Process arguments -^^^^^^^^^^^^^^^^^ - -Networking -^^^^^^^^^^ - -Process namespacing -^^^^^^^^^^^^^^^^^^^ - -Resource limits -^^^^^^^^^^^^^^^ - -Process monitoring -^^^^^^^^^^^^^^^^^^ - -Logging -^^^^^^^ - -Signals -^^^^^^^ - -Pseudo-terminal allocation -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Security -^^^^^^^^ - From b0bd5c4e73bce08fe90dcb32da06427da5ac82b5 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:40:55 -0400 Subject: [PATCH 17/54] Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304 Upstream-commit: bdc79ac8b2dfad302f9e144711067a566726cfa2 Component: engine --- components/engine/Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index aadabb8711..7258af5bf7 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -20,6 +20,8 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " + # Listen on all interfaces so that the daemon is accessible from the host + pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From cd3a1b485585f13de208d23b5e546adec9603ab3 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 26 Jul 2013 15:44:06 -0400 Subject: [PATCH 18/54] Update AUTHORS Upstream-commit: 5eb590e79d081ecfa82ba940ea856c4a2c0411ca Component: engine --- components/engine/.mailmap | 2 ++ components/engine/AUTHORS | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/components/engine/.mailmap b/components/engine/.mailmap index 452ac41d8f..11ff5357d8 100644 --- a/components/engine/.mailmap +++ b/components/engine/.mailmap @@ -23,3 +23,5 @@ Thatcher Peskens Walter Stanish Roberto Hashioka +Konstantin Pelykh +David Sissitka diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index 86d03f6e12..c811de6526 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -4,12 +4,15 @@ # For a list of active project maintainers, see the MAINTAINERS file. # Al Tobey +Alex Gaynor Alexey Shamrin Andrea Luzzardi Andreas Tiefenthaler Andrew Munsell +Andrews Medina Andy Rothfusz Andy Smith +Anthony Bishopric Antony Messerli Barry Allard Brandon Liu @@ -23,17 +26,22 @@ Daniel Gasienica Daniel Mizyrycki Daniel Robinson Daniel Von Fange +Daniel YC Lin +David Sissitka Dominik Honnef Don Spaulding Dr Nic Williams Elias Probst Eric Hanchrow -Evan Wies Eric Myhre +Erno Hopearuoho +Evan Wies ezbercih +Fabrizio Regini Flavio Castelli Francisco Souza Frederick F. Kautz IV +Gabriel Monroy Gareth Rushgrove Guillaume J. Charmes Harley Laue @@ -41,6 +49,7 @@ Hunter Blanks Jeff Lindsay Jeremy Grosser Joffrey F +Johan Euphrosine John Costa Jon Wedaman Jonas Pfenniger @@ -48,28 +57,39 @@ Jonathan Rudenberg Joseph Anthony Pasquale Holsten Julien Barbier Jérôme Petazzoni +Karan Lyons +Keli Hu Ken Cochrane Kevin J. Lynagh kim0 +Kimbro Staken Kiran Gangadharan +Konstantin Pelykh Louis Opter +Marco Hennings Marcus Farkas Mark McGranaghan Maxim Treskin meejah Michael Crosby +Mike Gaffney Mikhail Sobolev +Nan Monnand Deng Nate Jones Nelson Chen Niall O'Higgins +Nick Stenning +Nick Stinemates odk- Paul Bowsher Paul Hammond Phil Spitler Piotr Bogdan Renato Riccieri Santos Zannon +Rhys Hiltner Robert Obryk Roberto Hashioka +Ryan Fowler Sam Alba Sam J Sharpe Shawn Siefkas @@ -83,6 +103,8 @@ Thomas Hansen Tianon Gravi Tim Terhorst Tobias Bieniek +Tobias Schwab +Tom Hulihan unclejack Victor Vieux Vivek Agarwal From bdd8af79196724d7ddfbef579bd46ec292c6dd3e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 26 Jul 2013 14:57:16 -0700 Subject: [PATCH 19/54] - Builder: Create directories with 755 instead of 700 within ADD instruction Upstream-commit: b15cfd3530cc228dc065746c9323758c8abb0481 Component: engine --- components/engine/buildfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 75ebdd7a7c..a25b86d794 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -242,7 +242,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } else if err := UntarPath(origPath, destPath); err != nil { utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file - if err := os.MkdirAll(path.Dir(destPath), 0700); err != nil { + if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := CopyWithTar(origPath, destPath); err != nil { From 3fa2b3c80a598c17233759282718060e16740f84 Mon Sep 17 00:00:00 2001 From: Mike Gaffney Date: Fri, 26 Jul 2013 18:29:27 -0700 Subject: [PATCH 20/54] Add required go version for compilation Upstream-commit: 2d85a20c71c9418301c3161c6853a07fca612676 Component: engine --- components/engine/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/README.md b/components/engine/README.md index 96da13feaf..5e03b2993c 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -167,7 +167,7 @@ Installing from source ---------------------- 1. Make sure you have a [Go language](http://golang.org/doc/install) -compiler and [git](http://git-scm.com) installed. +compiler >= 1.1 and [git](http://git-scm.com) installed. 2. Checkout the source code ```bash From fba18c65a3185bea2bb32ef461b57e3aff48c25c Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:00:36 -0700 Subject: [PATCH 21/54] Do not show empty parenthesis if the default configuration is missing. Upstream-commit: d4f70397930b15c8860a71df1abb9bc5ac9ab4ff Component: engine --- components/engine/commands.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 0fabaa385f..a7df369e5f 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -314,13 +314,21 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) + var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + if configDefault == "" { + fmt.Fprintf(cli.out, "%s: ", prompt) + } else { + fmt.Fprintf(cli.out, "%s (%s): ", prompt, configDefault) + } + } + authconfig, ok := cli.configFile.Configs[auth.IndexServerAddress()] if !ok { authconfig = auth.AuthConfig{} } if *flUsername == "" { - fmt.Fprintf(cli.out, "Username (%s): ", authconfig.Username) + promptDefault(cli.out, "Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -340,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - fmt.Fprintf(cli.out, "Email (%s): ", authconfig.Email) + promptDefault(cli.out, "Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 86b74909af3248f1cb0bcf103445df71baed118c Mon Sep 17 00:00:00 2001 From: David Calavera Date: Sat, 27 Jul 2013 10:17:57 -0700 Subject: [PATCH 22/54] Remove unused argument. Upstream-commit: 88b6ea993d76fb8891ee7a7fa8828b5c5753c7f5 Component: engine --- components/engine/commands.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index a7df369e5f..7288b58229 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -314,7 +314,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) - var promptDefault = func(stdout io.Writer, prompt string, configDefault string) { + var promptDefault = func(prompt string, configDefault string) { if configDefault == "" { fmt.Fprintf(cli.out, "%s: ", prompt) } else { @@ -328,7 +328,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flUsername == "" { - promptDefault(cli.out, "Username", authconfig.Username) + promptDefault("Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -348,7 +348,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - promptDefault(cli.out, "Email", authconfig.Email) + promptDefault("Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email From 730b8810f90e162ce6969e977be10fcee945aae5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sun, 28 Jul 2013 12:57:09 -0700 Subject: [PATCH 23/54] Remove deprecated copy from README Upstream-commit: 97a2dc96f23b25bc7980d2a4514b7065ff4edb9e Component: engine --- components/engine/README.md | 45 ------------------------------------- 1 file changed, 45 deletions(-) diff --git a/components/engine/README.md b/components/engine/README.md index 5e03b2993c..5eee1a3db6 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -444,51 +444,6 @@ With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - - -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - ### Legal Transfers of Docker shall be in accordance with applicable export From 183aa3c2d9a6e42e18f19f9c4219fb7775b6a957 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:15:27 +0000 Subject: [PATCH 24/54] use ParseRepositoryTag instead on split on : in imagedelete Upstream-commit: 63876e7dbdd372745855345632af62b8cd976dfa Component: engine --- components/engine/server.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..f416d8f06d 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1037,13 +1037,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - var tag string - if strings.Contains(name, ":") { - nameParts := strings.Split(name, ":") - name = nameParts[0] - tag = nameParts[1] - } - + name, tag := utils.ParseRepositoryTag(name) return srv.deleteImage(img, name, tag) } From 77e59c214867e93be8aacf505d35acc9aa0be941 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:01 +0000 Subject: [PATCH 25/54] add ParseRepositoryTag tests Upstream-commit: 3852d0599097581d5dc0bcfcb7aa010d564beb9a Component: engine --- components/engine/utils/utils.go | 7 +++---- components/engine/utils/utils_test.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index acb015becd..c70e80b72e 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -611,11 +611,11 @@ type JSONMessage struct { Status string `json:"status,omitempty"` Progress string `json:"progress,omitempty"` Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } @@ -631,7 +631,6 @@ func (jm *JSONMessage) Display(out io.Writer) (error) { return nil } - type StreamFormatter struct { json bool used bool diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 5caa809f67..5c480b9438 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -282,3 +282,24 @@ func TestParseHost(t *testing.T) { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } } + +func TestParseRepositoryTag(t *testing.T) { + if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) + } +} From c1c0668f041a479eff88225c7926a0c3b14a9250 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 12:16:14 +0000 Subject: [PATCH 26/54] add regression tests Upstream-commit: bb241c10e2f124406f00c8e40e00fca67934638a Component: engine --- components/engine/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 8612b3fcea..0caf8a5f24 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -20,7 +20,7 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestContainerTagImageDelete(t *testing.T) { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { t.Fatal(err) } From 224e45118c5680834731d0624d6877543f3273d5 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 29 Jul 2013 16:40:35 +0000 Subject: [PATCH 27/54] fix #1314 discard error when loading old container format Upstream-commit: b2aa877bf0a3f140516d1c17117369b2194f2b91 Component: engine --- components/engine/container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/container.go b/components/engine/container.go index d0b6ca4ce2..cbcd3f17fa 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -266,7 +266,8 @@ func (container *Container) FromDisk() error { return err } // Load container settings - if err := json.Unmarshal(data, container); err != nil { + // udp broke compat of docker.PortMapping, but it's not used when loading a container, we can skip it + if err := json.Unmarshal(data, container); err != nil && !strings.Contains(err.Error(), "docker.PortMapping") { return err } return nil From 1ccfbf8698584ad97e67a104b38d9bba7a89155f Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Mon, 29 Jul 2013 09:45:19 -0700 Subject: [PATCH 28/54] testing, issue #1331: Add registry functional test to docker-ci Upstream-commit: 17ffb0ac84874656a61a477fa29048cb82b28748 Component: engine --- components/engine/testing/README.rst | 4 ++++ components/engine/testing/Vagrantfile | 4 +++- .../engine/testing/buildbot/credentials.cfg | 5 +++++ components/engine/testing/buildbot/master.cfg | 20 +++++++++++++++++-- .../engine/testing/buildbot/requirements.txt | 1 + .../testing/buildbot/setup_credentials.sh | 17 ++++++++++++++++ .../testing/functionaltests/test_registry.sh | 11 ++++++++++ 7 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 components/engine/testing/buildbot/credentials.cfg create mode 100755 components/engine/testing/buildbot/setup_credentials.sh create mode 100755 components/engine/testing/functionaltests/test_registry.sh diff --git a/components/engine/testing/README.rst b/components/engine/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/components/engine/testing/README.rst +++ b/components/engine/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/components/engine/testing/Vagrantfile +++ b/components/engine/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/components/engine/testing/buildbot/credentials.cfg b/components/engine/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/components/engine/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/components/engine/testing/buildbot/master.cfg b/components/engine/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/components/engine/testing/buildbot/master.cfg +++ b/components/engine/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/components/engine/testing/buildbot/requirements.txt b/components/engine/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/components/engine/testing/buildbot/requirements.txt +++ b/components/engine/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/components/engine/testing/buildbot/setup_credentials.sh b/components/engine/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/components/engine/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/components/engine/testing/functionaltests/test_registry.sh b/components/engine/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/components/engine/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow From 4b37d9ec34f816bd7fb8a1ad22ea0a2703016091 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 10:28:41 -0700 Subject: [PATCH 29/54] Remove unnecessary signal conditional. Upstream-commit: c8ec36d1b9bfbe1e22acd0124409ecb5a109d406 Component: engine --- components/engine/commands.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 0fabaa385f..b3e7976995 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1658,14 +1658,11 @@ func (cli *DockerCli) monitorTtySize(id string) error { } cli.resizeTty(id) - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGWINCH) + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGWINCH) go func() { - for sig := range c { - if sig == syscall.SIGWINCH { - cli.resizeTty(id) - } - } + <-sigchan + cli.resizeTty(id) }() return nil } From dc883510210f5a92f0a21ad468df47e382116799 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 29 Jul 2013 11:03:09 -0700 Subject: [PATCH 30/54] Make sure ADD will create everything in 0755 Upstream-commit: f7542664e3efb35b6b153dc2b2cc9cdac181989a Component: engine --- components/engine/archive.go | 2 +- components/engine/docs/sources/use/builder.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/archive.go b/components/engine/archive.go index 01af86006f..bb79cd34d4 100644 --- a/components/engine/archive.go +++ b/components/engine/archive.go @@ -173,7 +173,7 @@ func CopyWithTar(src, dst string) error { } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) - if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index aa2fd6b92a..9f5ee8b795 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -182,7 +182,7 @@ The copy obeys the following rules: written at ````. * If ```` doesn't exist, it is created along with all missing directories in its path. All new files and directories are created - with mode 0700, uid and gid 0. + with mode 0755, uid and gid 0. 3.8 ENTRYPOINT -------------- From 4dd9531b175f79d5bfaa9cbb9582027850ec9b7e Mon Sep 17 00:00:00 2001 From: David Calavera Date: Mon, 29 Jul 2013 11:13:59 -0700 Subject: [PATCH 31/54] Keep the loop to allow resizing more than once. Upstream-commit: 10e37198aa14cb3192fd0bf29572a5dce58c348f Component: engine --- components/engine/commands.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index b3e7976995..b4853e967d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1661,8 +1661,10 @@ func (cli *DockerCli) monitorTtySize(id string) error { sigchan := make(chan os.Signal, 1) signal.Notify(sigchan, syscall.SIGWINCH) go func() { - <-sigchan - cli.resizeTty(id) + for { + <-sigchan + cli.resizeTty(id) + } }() return nil } From e9895c9839e36227492d8394557c3c7cd4572d8a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 17 Jul 2013 12:13:22 -0700 Subject: [PATCH 32/54] Refactor checksum Upstream-commit: 8ca7b0646e6c4346075656f46847f53c2e868a3d Component: engine --- components/engine/auth/auth.go | 4 +- components/engine/graph.go | 61 +--------- components/engine/image.go | 99 ---------------- components/engine/registry/registry.go | 58 +++++++-- components/engine/server.go | 96 +++------------ components/engine/utils/tarsum.go | 155 +++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 247 deletions(-) create mode 100644 components/engine/utils/tarsum.go diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index 39de876875..e402031ca2 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -const INDEXSERVER = "https://index.docker.io/v1/" +//const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/" +const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/components/engine/graph.go b/components/engine/graph.go index 42d1bdbd4c..eea2cbec8a 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -1,9 +1,7 @@ package docker import ( - "encoding/json" "fmt" - "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -11,17 +9,13 @@ import ( "path" "path/filepath" "strings" - "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *utils.TruncIndex - checksumLock map[string]*sync.Mutex - lockSumFile *sync.Mutex - lockSumMap *sync.Mutex + Root string + idIndex *utils.TruncIndex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -36,11 +30,8 @@ func NewGraph(root string) (*Graph, error) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: utils.NewTruncIndex(), - checksumLock: make(map[string]*sync.Mutex), - lockSumFile: &sync.Mutex{}, - lockSumMap: &sync.Mutex{}, + Root: abspath, + idIndex: utils.NewTruncIndex(), } if err := graph.restore(); err != nil { return nil, err @@ -99,11 +90,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, err } } - graph.lockSumMap.Lock() - defer graph.lockSumMap.Unlock() - if _, exists := graph.checksumLock[img.ID]; !exists { - graph.checksumLock[img.ID] = &sync.Mutex{} - } return img, nil } @@ -126,7 +112,6 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut if err := graph.Register(layerData, layerData != nil, img); err != nil { return nil, err } - go img.Checksum() return img, nil } @@ -154,7 +139,6 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { } img.graph = graph graph.idIndex.Add(img.ID) - graph.checksumLock[img.ID] = &sync.Mutex{} return nil } @@ -311,40 +295,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) getStoredChecksums() (map[string]string, error) { - checksums := make(map[string]string) - // FIXME: Store the checksum in memory - - if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil { - if err := json.Unmarshal(checksumDict, &checksums); err != nil { - return nil, err - } - } - return checksums, nil -} - -func (graph *Graph) storeChecksums(checksums map[string]string) error { - checksumJSON, err := json.Marshal(checksums) - if err != nil { - return err - } - if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJSON, 0600); err != nil { - return err - } - return nil -} - -func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error { - graph.lockSumFile.Lock() - defer graph.lockSumFile.Unlock() - - localChecksums, err := graph.getStoredChecksums() - if err != nil { - return err - } - for id, elem := range newChecksums { - localChecksums[id] = elem.Checksum - } - return graph.storeChecksums(localChecksums) -} diff --git a/components/engine/image.go b/components/engine/image.go index 5240ec776f..dd066f88cd 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -2,7 +2,6 @@ package docker import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -72,26 +71,6 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { return err } - if store { - layerArchive := layerArchivePath(root) - file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - // FIXME: Retrieve the image layer size from here? - if _, err := io.Copy(file, layerData); err != nil { - return err - } - // FIXME: Don't close/open, read/write instead of Copy - file.Close() - - file, err = os.Open(layerArchive) - if err != nil { - return err - } - defer file.Close() - layerData = file - } // If layerData is not nil, unpack it into the new layer if layerData != nil { start := time.Now() @@ -128,10 +107,6 @@ func layerPath(root string) string { return path.Join(root, "layer") } -func layerArchivePath(root string) string { - return path.Join(root, "layer.tar.xz") -} - func jsonPath(root string) string { return path.Join(root, "json") } @@ -308,80 +283,6 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) Checksum() (string, error) { - img.graph.checksumLock[img.ID].Lock() - defer img.graph.checksumLock[img.ID].Unlock() - - root, err := img.root() - if err != nil { - return "", err - } - - checksums, err := img.graph.getStoredChecksums() - if err != nil { - return "", err - } - if checksum, ok := checksums[img.ID]; ok { - return checksum, nil - } - - layer, err := img.layer() - if err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) - if err != nil { - return "", err - } - - var layerData io.Reader - - if file, err := os.Open(layerArchivePath(root)); err != nil { - if os.IsNotExist(err) { - layerData, err = Tar(layer, Xz) - if err != nil { - return "", err - } - } else { - return "", err - } - } else { - defer file.Close() - layerData = file - } - - h := sha256.New() - if _, err := h.Write(jsonData); err != nil { - return "", err - } - if _, err := h.Write([]byte("\n")); err != nil { - return "", err - } - - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - - // Reload the json file to make sure not to overwrite faster sums - img.graph.lockSumFile.Lock() - defer img.graph.lockSumFile.Unlock() - - checksums, err = img.graph.getStoredChecksums() - if err != nil { - return "", err - } - - checksums[img.ID] = hash - - // Dump the checksums to disc - if err := img.graph.storeChecksums(checksums); err != nil { - return hash, err - } - - return hash, nil -} - func (img *Image) getParentsSize(size int64) int64 { parentImage, err := img.GetParent() if err != nil || parentImage == nil { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index adef1c7baa..cac77ba049 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -330,16 +330,52 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e }, nil } +func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + // Push a local image to the registry func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - // FIXME: try json with UTF8 + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.Header.Set("X-Docker-Checksum", imgData.Checksum) r.setUserAgent(req) utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) @@ -364,10 +400,14 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { - return err + return "", err } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} @@ -375,18 +415,18 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) + return "", fmt.Errorf("Failed to upload layer: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return nil + return tarsumLayer.Sum(), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { diff --git a/components/engine/server.go b/components/engine/server.go index ce1fc8eaf8..6944df315f 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -455,12 +455,6 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName return err } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err - } - utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { @@ -598,41 +592,6 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return nil } -// Retrieve the checksum of an image -// Priority: -// - Check on the stored checksums -// - Check if the archive exists, if it does not, ask the registry -// - If the archive does exists, process the checksum from it -// - If the archive does not exists and not found on registry, process checksum from layer -func (srv *Server) getChecksum(imageID string) (string, error) { - // FIXME: Use in-memory map instead of reading the file each time - if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { - return "", err - } else if checksum, exists := sums[imageID]; exists { - return checksum, nil - } - - img, err := srv.runtime.graph.Get(imageID) - if err != nil { - return "", err - } - - if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil { - if os.IsNotExist(err) { - // TODO: Ask the registry for the checksum - // As the archive is not there, it is supposed to come from a pull. - } else { - return "", err - } - } - - checksum, err := img.Checksum() - if err != nil { - return "", err - } - return checksum, nil -} - // Retrieve the all the images to be uploaded in the correct order // Note: we can't use a map as it is not ordered func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) { @@ -649,14 +608,10 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return nil } imageSet[img.ID] = struct{}{} - checksum, err := srv.getChecksum(img.ID) - if err != nil { - return err - } + imgList = append([]*registry.ImgData{{ - ID: img.ID, - Checksum: checksum, - Tag: tag, + ID: img.ID, + Tag: tag, }}, imgList...) return nil }) @@ -666,7 +621,7 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - out.Write(sf.FormatStatus("Processing checksums")) + imgList, err := srv.getImageList(localRepo) if err != nil { return err @@ -716,14 +671,8 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } out.Write(sf.FormatStatus("Pushing %s", imgID)) - // Make sure we have the image's checksum - checksum, err := srv.getChecksum(imgID) - if err != nil { - return err - } imgData := ®istry.ImgData{ - ID: imgID, - Checksum: checksum, + ID: imgID, } // Send the json @@ -735,36 +684,23 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, return err } - // Retrieve the tarball to be sent - var layerData *TempArchive - // If the archive exists, use it - file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID))) + layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - if os.IsNotExist(err) { - // If the archive does not exist, create one from the layer - layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - } else { - return err - } - } else { - defer file.Close() - st, err := file.Stat() - if err != nil { - return err - } - layerData = &TempArchive{ - File: file, - Size: st.Size(), - } + return fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + return err + } else { + imgData.Checksum = checksum + } + + // Send the checksum + if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { return err } + return nil } diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go new file mode 100644 index 0000000000..0fd5ac106b --- /dev/null +++ b/components/engine/utils/tarsum.go @@ -0,0 +1,155 @@ +package utils + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "hash" + "io" + "sort" + "strconv" +) + +type verboseHash struct { + hash.Hash +} + +func (h verboseHash) Write(buf []byte) (int, error) { + Debugf("--->%s<---", buf) + return h.Hash.Write(buf) +} + +type TarSum struct { + io.Reader + tarR *tar.Reader + tarW *tar.Writer + gz *gzip.Writer + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + h2 verboseHash + sums []string + finished bool + first bool +} + +func (ts *TarSum) encodeHeader(h *tar.Header) error { + for _, elem := range [][2]string{ + {"name", h.Name}, + {"mode", strconv.Itoa(int(h.Mode))}, + {"uid", strconv.Itoa(h.Uid)}, + {"gid", strconv.Itoa(h.Gid)}, + {"size", strconv.Itoa(int(h.Size))}, + {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"typeflag", string([]byte{h.Typeflag})}, + {"linkname", h.Linkname}, + {"uname", h.Uname}, + {"gname", h.Gname}, + {"devmajor", strconv.Itoa(int(h.Devmajor))}, + {"devminor", strconv.Itoa(int(h.Devminor))}, + // {"atime", strconv.Itoa(int(h.AccessTime.UTC().Unix()))}, + // {"ctime", strconv.Itoa(int(h.ChangeTime.UTC().Unix()))}, + } { + // Debugf("-->%s<-- -->%s<--", elem[0], elem[1]) + if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { + return err + } + } + return nil +} + +func (ts *TarSum) Read(buf []byte) (int, error) { + if ts.gz == nil { + ts.bufTar = bytes.NewBuffer([]byte{}) + ts.bufGz = bytes.NewBuffer([]byte{}) + ts.tarR = tar.NewReader(ts.Reader) + ts.tarW = tar.NewWriter(ts.bufTar) + ts.gz = gzip.NewWriter(ts.bufGz) + ts.h = sha256.New() + // ts.h = verboseHash{sha256.New()} + ts.h.Reset() + ts.first = true + } + + if ts.finished { + return ts.bufGz.Read(buf) + } + buf2 := make([]byte, len(buf), cap(buf)) + + n, err := ts.tarR.Read(buf2) + if err != nil { + if err == io.EOF { + if _, err := ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + if !ts.first { + ts.sums = append(ts.sums, hex.EncodeToString(ts.h.Sum(nil))) + ts.h.Reset() + } else { + ts.first = false + } + + currentHeader, err := ts.tarR.Next() + if err != nil { + if err == io.EOF { + if err := ts.gz.Close(); err != nil { + return 0, err + } + ts.finished = true + return n, nil + } + return n, err + } + if err := ts.encodeHeader(currentHeader); err != nil { + return 0, err + } + if err := ts.tarW.WriteHeader(currentHeader); err != nil { + return 0, err + } + if _, err := ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + if _, err := io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) + } + return n, err + } + + // Filling the hash buffer + if _, err = ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + + // Filling the tar writter + if _, err = ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + + // Filling the gz writter + if _, err = io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) +} + +func (ts *TarSum) Sum() string { + sort.Strings(ts.sums) + h := sha256.New() + for _, sum := range ts.sums { + Debugf("-->%s<--", sum) + h.Write([]byte(sum)) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + Debugf("checksum processed: %s", checksum) + return checksum +} From 0d6f4c9ed448770f52cc0be7785237520b4eaca1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 14:50:32 -0700 Subject: [PATCH 33/54] Handle extra-paremeter within checksum calculations Upstream-commit: e3f68b22d8f0635a8c08ab56721e56dbe570a49a Component: engine --- components/engine/registry/registry.go | 17 +++++++++++------ components/engine/server.go | 2 +- components/engine/utils/tarsum.go | 7 +++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index cac77ba049..40b9872a4a 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -17,8 +17,10 @@ import ( "strings" ) -var ErrAlreadyExists = errors.New("Image already exists") -var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +var ( + ErrAlreadyExists = errors.New("Image already exists") + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +) func pingRegistryEndpoint(endpoint string) error { if endpoint == auth.IndexServerAddress() { @@ -266,8 +268,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err @@ -378,7 +383,6 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) r.setUserAgent(req) - utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -400,11 +404,12 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) { +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") tarsumLayer := &utils.TarSum{Reader: layer} + req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) if err != nil { return "", err @@ -426,7 +431,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr } return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) } - return tarsumLayer.Sum(), nil + return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { @@ -474,7 +479,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("PUT %s", u) + utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { diff --git a/components/engine/server.go b/components/engine/server.go index 6944df315f..7309279805 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -690,7 +690,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, } // Send the layer - if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { return err } else { imgData.Checksum = checksum diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 0fd5ac106b..015b9b3076 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -142,14 +142,17 @@ func (ts *TarSum) Read(buf []byte) (int, error) { return ts.bufGz.Read(buf) } -func (ts *TarSum) Sum() string { +func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - checksum := "tarsum+sha256:" + hex.EncodeToString(ts.h.Sum(nil)) + if extra != nil { + h.Write(extra) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum } From 78c57a64f00c90de82b3a766eb89c502151f7872 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:40:33 -0700 Subject: [PATCH 34/54] Refactor the image size storage Upstream-commit: 0badda9f1587c11a13dca17c68b30addd757237c Component: engine --- components/engine/graph.go | 6 +++--- components/engine/image.go | 43 +++++++++++++++++++++++++++++-------- components/engine/server.go | 2 +- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index eea2cbec8a..1fe14f6458 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { return err } // Commit diff --git a/components/engine/image.go b/components/engine/image.go index dd066f88cd..7f03cc4bf8 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -13,6 +13,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "time" ) @@ -46,6 +47,19 @@ func LoadImage(root string) (*Image, error) { if err := ValidateID(img.ID); err != nil { return nil, err } + + if buf, err := ioutil.ReadFile(path.Join(root, "layersize")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + if size, err := strconv.Atoi(string(buf)); err != nil { + return nil, err + } else { + img.Size = int64(size) + } + } + // Check that the filesystem layer exists if stat, err := os.Stat(layerPath(root)); err != nil { if os.IsNotExist(err) { @@ -58,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) @@ -81,25 +95,36 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds()) } + // If raw json is provided, then use it + if jsonData != nil { + return ioutil.WriteFile(jsonPath(root), jsonData, 0600) + } else { // Otherwise, unmarshal the image + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } + return StoreSize(img, root) } func StoreSize(img *Image, root string) error { layer := layerPath(root) + var totalSize int64 = 0 filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error { - img.Size += fileInfo.Size() + totalSize += fileInfo.Size() return nil }) + img.Size = totalSize - // Store the json ball - jsonData, err := json.Marshal(img) - if err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err + if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil { + return nil } + return nil } diff --git a/components/engine/server.go b/components/engine/server.go index 7309279805..de73d6c815 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { return err } } From c364da0b5142fa2e10773de9e3e651112fd662e4 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 15:44:55 -0700 Subject: [PATCH 35/54] Remove unused parameter Upstream-commit: 0f134b4bf81a4d0160932852854b190b7ee7e3b9 Component: engine --- components/engine/graph.go | 6 +++--- components/engine/image.go | 2 +- components/engine/server.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/graph.go b/components/engine/graph.go index 1fe14f6458..bf27f93ed4 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -109,7 +109,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(nil, layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, img); err != nil { return nil, err } return img, nil @@ -117,7 +117,7 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -130,7 +130,7 @@ func (graph *Graph) Register(jsonData []byte, layerData Archive, store bool, img if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, jsonData, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp); err != nil { return err } // Commit diff --git a/components/engine/image.go b/components/engine/image.go index 7f03cc4bf8..220f1c70a5 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -72,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) diff --git a/components/engine/server.go b/components/engine/server.go index de73d6c815..aecc730794 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -439,7 +439,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return err } defer layer.Close() - if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), img); err != nil { return err } } From 0e6bb26f7715b300bd7a152b7da15092df6336d8 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:16:31 -0700 Subject: [PATCH 36/54] Switch json/payload order Upstream-commit: 394941b6b0a30fecf8ae7b6de5880fa553141f93 Component: engine --- components/engine/auth/auth.go | 4 ++-- components/engine/utils/tarsum.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index e402031ca2..6dd6ceb620 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -16,9 +16,9 @@ import ( const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation -//const INDEXSERVER = "https://index.docker.io/v1/" +const INDEXSERVER = "https://index.docker.io/v1/" -const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" +//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 015b9b3076..d3e1db61f1 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -145,13 +145,13 @@ func (ts *TarSum) Read(buf []byte) (int, error) { func (ts *TarSum) Sum(extra []byte) string { sort.Strings(ts.sums) h := sha256.New() + if extra != nil { + h.Write(extra) + } for _, sum := range ts.sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } - if extra != nil { - h.Write(extra) - } checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) Debugf("checksum processed: %s", checksum) return checksum From 753746a54d821b03cdaa9b1bcf6d04904e2f7427 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 22 Jul 2013 16:44:34 -0700 Subject: [PATCH 37/54] Make sure the index also receives the checksums Upstream-commit: 5b27652ac6eaf1bc4c2a16e51919ec4272a58fd6 Component: engine --- components/engine/registry/registry.go | 14 +++++++++++++- components/engine/server.go | 22 ++++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 40b9872a4a..4e9dd8895f 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -469,7 +469,19 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - imgListJSON, err := json.Marshal(imgList) + cleanImgList := []*ImgData{} + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) if err != nil { return nil, err } diff --git a/components/engine/server.go b/components/engine/server.go index aecc730794..88ff2ac3a6 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -645,9 +645,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { + if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err + } else { + elem.Checksum = checksum } out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { @@ -663,11 +665,11 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName return nil } -func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) + return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } out.Write(sf.FormatStatus("Pushing %s", imgID)) @@ -679,29 +681,29 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) - return nil + return "", nil } - return err + return "", err } layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) + return "", fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token, jsonRaw); err != nil { - return err + return "", err } else { imgData.Checksum = checksum } // Send the checksum if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { - return err + return "", err } - return nil + return imgData.Checksum, nil } // FIXME: Allow to interupt current push when new push of same image is done. @@ -739,7 +741,7 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo var token []string out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) - if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { + if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil From 7e99f4ac78242ffb92e6778514ff7d85a83832c3 Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Mon, 29 Jul 2013 14:17:15 -0700 Subject: [PATCH 38/54] Updated the description of run -d The goal is to make it more clear this will give you the container id after run completes. Since stdout is now standard on run, "docker run -d" is the best (or only) way to get the container ID returned from docker after a plain run, but the description (help) does not hint any such thing. Upstream-commit: 5dc86d7bca17c2996264a18cc26f06d30e532588 Component: engine --- components/engine/container.go | 2 +- components/engine/docs/sources/commandline/command/run.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index d0b6ca4ce2..d610c3c7d4 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -100,7 +100,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, flHostname := cmd.String("h", "", "Container host name") flUser := cmd.String("u", "", "Username or UID") - flDetach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id") flAttach := NewAttachOpts() cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached") diff --git a/components/engine/docs/sources/commandline/command/run.rst b/components/engine/docs/sources/commandline/command/run.rst index db67ef0705..db043c3b3f 100644 --- a/components/engine/docs/sources/commandline/command/run.rst +++ b/components/engine/docs/sources/commandline/command/run.rst @@ -15,7 +15,7 @@ -a=map[]: Attach to stdin, stdout or stderr. -c=0: CPU shares (relative weight) -cidfile="": Write the container ID to the file - -d=false: Detached mode: leave the container running in the background + -d=false: Detached mode: Run container in the background, print new container id -e=[]: Set environment variables -h="": Container host name -i=false: Keep stdin open even if not attached From 86d430ea320a9d5c165d430544ba05f90b2305bb Mon Sep 17 00:00:00 2001 From: dsissitka Date: Tue, 30 Jul 2013 01:39:29 -0400 Subject: [PATCH 39/54] Fixed a couple of minor syntax errors. Upstream-commit: 9ba998312de5826f24f693c8518ecda700135b4b Component: engine --- .../engine/docs/sources/examples/couchdb_data_volumes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/examples/couchdb_data_volumes.rst b/components/engine/docs/sources/examples/couchdb_data_volumes.rst index d6babe557f..97af733a82 100644 --- a/components/engine/docs/sources/examples/couchdb_data_volumes.rst +++ b/components/engine/docs/sources/examples/couchdb_data_volumes.rst @@ -39,7 +39,7 @@ This time, we're requesting shared access to $COUCH1's volumes. .. code-block:: bash - COUCH2=$(docker run -d -volumes-from $COUCH1) shykes/couchdb:2013-05-03) + COUCH2=$(docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03) Browse data on the second database ---------------------------------- @@ -48,6 +48,6 @@ Browse data on the second database HOST=localhost URL="http://$HOST:$(docker port $COUCH2 5984)/_utils/" - echo "Navigate to $URL in your browser. You should see the same data as in the first database!" + echo "Navigate to $URL in your browser. You should see the same data as in the first database"'!' Congratulations, you are running 2 Couchdb containers, completely isolated from each other *except* for their data. From fb6e560276b5e8f6bd32f112d4e3bbdd39302c53 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:47:29 +0200 Subject: [PATCH 40/54] add ufw doc Upstream-commit: 7d0b8c726c0d178291063751ac735d6caebfb45a Component: engine --- .../docs/sources/installation/ubuntulinux.rst | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index ed592d3a9d..299b7a29e9 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -19,6 +19,8 @@ Docker has the following dependencies * Linux kernel 3.8 (read more about :ref:`kernel`) * AUFS file system support (we are working on BTRFS support as an alternative) +Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ + .. _ubuntu_precise: Ubuntu Precise 12.04 (LTS) (64-bit) @@ -135,3 +137,35 @@ Verify it worked **Done!**, now continue with the :ref:`hello_world` example. + + +.. _ufw: + +Docker and UFW +^^^^^^^^^^^^^^ + +Docker uses a bridge to manage containers networking, by default UFW drop all `forwarding`, a first step is to enable forwarding: + +.. code-block:: bash + + sudo nano /etc/default/ufw + ---- + # Change: + # DEFAULT_FORWARD_POLICY="DROP" + # to + DEFAULT_FORWARD_POLICY="ACCEPT" + +Then reload UFW: + +.. code-block:: bash + + sudo ufw reload + + +UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, +you should allow incoming connexions on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + From 6c28901fdbf8d8547cd76a21e0a38849f346fa46 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 13:13:18 +0000 Subject: [PATCH 41/54] fix tests about refactor checksums Upstream-commit: b14c251862021b2bc82aa8e0146e5e5e80f1c713 Component: engine --- components/engine/graph_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/graph_test.go b/components/engine/graph_test.go index 18682338d9..2898fccf99 100644 --- a/components/engine/graph_test.go +++ b/components/engine/graph_test.go @@ -38,7 +38,7 @@ func TestInterruptedRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - go graph.Register(badArchive, false, image) + go graph.Register(nil, badArchive, image) time.Sleep(200 * time.Millisecond) w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) if _, err := graph.Get(image.ID); err == nil { @@ -49,7 +49,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(goodArchive, false, image); err != nil { + if err := graph.Register(nil, goodArchive, image); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(archive, false, image) + err = graph.Register(nil, archive, image) if err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(archive, false, img1); err != nil { + if err := graph.Register(nil, archive, img1); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { From c7d245ec6d1824f5980ac9918eb086ed02cc4e1b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 16:39:35 +0000 Subject: [PATCH 42/54] Add check that the request is good Upstream-commit: e4752c8c1a09fc3cc96dbb9be7183b271db3d6b7 Component: engine --- components/engine/utils/utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 190dc175cd..def5ae5a50 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -736,6 +736,9 @@ func GetReleaseVersion() string { return "" } defer resp.Body.Close() + if resp.ContentLength > 24 || resp.StatusCode != 200 { + return "" + } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "" From 554c357a3ee4b84a513a3e7458c062c12714fb33 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Tue, 30 Jul 2013 17:18:19 +0000 Subject: [PATCH 43/54] update http://get.docker.io/latest Upstream-commit: e66e0289abc213164dca1e1eadfb0380b6e81904 Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 6050000582..dd365dc30e 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -50,6 +50,7 @@ release: $(BINRELEASE) s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz s3cmd -P put docker-latest.tgz s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-latest.tgz s3cmd -P put $(SRCRELEASE)/bin/docker s3://get.docker.io/builds/`uname -s`/`uname -m`/docker + echo $(RELEASE_VERSION) > latest ; s3cmd -P put latest s3://get.docker.io/latest ; rm latest srcrelease: $(SRCRELEASE) deps: $(DOCKER_DIR) @@ -65,7 +66,6 @@ $(BINRELEASE): $(SRCRELEASE) rm -f $(BINRELEASE) cd $(SRCRELEASE); make; cp -R bin docker-$(RELEASE_VERSION); tar -f ../$(BINRELEASE) -zv -c docker-$(RELEASE_VERSION) cd $(SRCRELEASE); cp -R bin docker-latest; tar -f ../docker-latest.tgz -zv -c docker-latest - clean: @rm -rf $(dir $(DOCKER_BIN)) ifeq ($(GOPATH), $(BUILD_DIR)) From 7aa4663949e8f79b0adb7e8f8d71126631c7596a Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 24 Jul 2013 03:01:24 +0000 Subject: [PATCH 44/54] Return registy status code in error Added Details map to the JSONMessage Upstream-commit: 3043c2641990d94298c6377b7ef14709263a4709 Component: engine --- components/engine/api.go | 12 +++++++---- components/engine/commands.go | 29 +++++++++++++++++--------- components/engine/registry/registry.go | 24 ++++++++++----------- components/engine/utils/error.go | 18 ++++++++++++++++ components/engine/utils/utils.go | 26 ++++++++++++++++------- 5 files changed, 75 insertions(+), 34 deletions(-) create mode 100644 components/engine/utils/error.go diff --git a/components/engine/api.go b/components/engine/api.go index 4ad2ba461a..5869669df0 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + w.Write(sf.FormatError(err, 0)) return nil } } @@ -472,7 +472,11 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - w.Write(sf.FormatError(err)) + var code int + if httpErr, ok := err.(*utils.HTTPRequestError); ok { + code = httpErr.StatusCode + } + w.Write(sf.FormatError(err, code)) return nil } return err diff --git a/components/engine/commands.go b/components/engine/commands.go index 95ddca1f1d..9ad2c367ae 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -30,7 +30,8 @@ import ( const VERSION = "0.5.0-dev" var ( - GITCOMMIT string + GITCOMMIT string + AuthRequiredError error = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { @@ -814,10 +815,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged("push"); err != nil { - return err - } - // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -826,13 +823,22 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.configFile.Configs[auth.IndexServerAddress()].Username, name) } - buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) - if err != nil { - return err + v := url.Values{} + push := func() error { + buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) + if err != nil { + return err + } + + return cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out) } - v := url.Values{} - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { + if err := push(); err != nil { + if err == AuthRequiredError { + if err = cli.checkIfLogged("push"); err == nil { + return push() + } + } return err } return nil @@ -1559,6 +1565,9 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e } else if err != nil { return err } + if jm.Error != nil && jm.Error.Code == 401 { + return AuthRequiredError + } jm.Display(out) } } else { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 4e9dd8895f..ed6f4c7df8 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -147,7 +147,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -197,7 +197,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -289,12 +289,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -391,7 +391,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } var jsonBody map[string]string if err := json.Unmarshal(errBody, &jsonBody); err != nil { @@ -399,7 +399,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis } else if jsonBody["error"] == "Image already exists" { return ErrAlreadyExists } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) } return nil } @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } @@ -463,7 +463,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } @@ -540,7 +540,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -564,7 +564,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -586,7 +586,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { diff --git a/components/engine/utils/error.go b/components/engine/utils/error.go new file mode 100644 index 0000000000..7e3c846ebc --- /dev/null +++ b/components/engine/utils/error.go @@ -0,0 +1,18 @@ +package utils + +import ( + "net/http" +) + +type HTTPRequestError struct { + Message string + StatusCode int +} + +func (e *HTTPRequestError) Error() string { + return e.Message +} + +func NewHTTPRequestError(msg string, resp *http.Response) error { + return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index c70e80b72e..659d960d0a 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -607,12 +607,22 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` + Status string `json:"status,omitempty"` + Progress string `json:"progress,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message } func (jm *JSONMessage) Display(out io.Writer) error { @@ -621,8 +631,8 @@ func (jm *JSONMessage) Display(out io.Writer) error { } if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != "" { - return fmt.Errorf(jm.Error) + } else if jm.Error != nil { + return jm.Error } else if jm.ID != "" { fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { @@ -656,7 +666,7 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil { + if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") From e5429eecead9274240eaf141d373665a1111cf16 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 07:56:53 +0000 Subject: [PATCH 45/54] improve tests Upstream-commit: 73c6d9f135493220f034d440d26fedc0242e133a Component: engine --- components/engine/server_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/components/engine/server_test.go b/components/engine/server_test.go index 0caf8a5f24..de95d743ba 100644 --- a/components/engine/server_test.go +++ b/components/engine/server_test.go @@ -20,7 +20,11 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } - if err := srv.runtime.repositories.Set("utest:5000/docker", "tag2", unitTestImageName, false); err != nil { + + if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { + t.Fatal(err) + } + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil { t.Fatal(err) } @@ -29,11 +33,24 @@ func TestContainerTagImageDelete(t *testing.T) { t.Fatal(err) } + if len(images) != len(initialImages)+3 { + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + } + + if _, err := srv.ImageDelete("utest/docker:tag2", true); err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + if len(images) != len(initialImages)+2 { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } - if _, err := srv.ImageDelete("utest:5000/docker:tag2", true); err != nil { + if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { t.Fatal(err) } From 496b0008be2bd1ac2abbe63c82e2cc95c978ef61 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 31 Jul 2013 08:01:20 +0000 Subject: [PATCH 46/54] fix same issue in api.go Upstream-commit: a7068510a5ee4af6776221ba00bc332266f97088 Component: engine --- components/engine/api.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index 834c41a68c..d0a8ada249 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -786,12 +786,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") - tag := "" - if strings.Contains(repoName, ":") { - remoteParts := strings.Split(repoName, ":") - tag = remoteParts[1] - repoName = remoteParts[0] - } + repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader From 30e6d541d0b27da785ccbb43ffb848adc62488b9 Mon Sep 17 00:00:00 2001 From: Nolan Date: Tue, 30 Jul 2013 13:23:34 -0500 Subject: [PATCH 47/54] Add hostname to the container environment. Upstream-commit: 9a604acc23d30d00ae907acdf756cbcdf0e4cf83 Component: engine --- components/engine/container.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/container.go b/components/engine/container.go index d610c3c7d4..ccc7ab3e9f 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -652,6 +652,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-e", "HOME=/", "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "-e", "container=lxc", + "-e", "HOSTNAME="+container.Config.Hostname, ) for _, elem := range container.Config.Env { From d69f94df65007e156adc9bc9fed2c65e1dd958aa Mon Sep 17 00:00:00 2001 From: Thatcher Peskens Date: Wed, 31 Jul 2013 12:17:42 -0700 Subject: [PATCH 48/54] Solved the logo being squished in Safari Upstream-commit: e0c24ccfc37b64eb919c5675d8a8f383201fa7cb Component: engine --- components/engine/docs/theme/docker/layout.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 0b22f22fab..ca26f44dc0 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -79,7 +79,7 @@
- +
From da0dfa9ebd66eff0cbd7708bcd77e21dca48936d Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Fri, 2 Aug 2013 12:24:38 +0700 Subject: [PATCH 49/54] Move note about officially supported kernel It seems this a general note about kernel issues and not specific to Cgroups or namespaces. Upstream-commit: 2424480e2ce14d848c041c53cc041aab91308081 Component: engine --- .../engine/docs/sources/installation/kernel.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/sources/installation/kernel.rst b/components/engine/docs/sources/installation/kernel.rst index 58730f8191..7c5715a62d 100644 --- a/components/engine/docs/sources/installation/kernel.rst +++ b/components/engine/docs/sources/installation/kernel.rst @@ -15,12 +15,11 @@ In short, Docker has the following kernel requirements: - Cgroups and namespaces must be enabled. - - The officially supported kernel is the one recommended by the - :ref:`ubuntu_linux` installation path. It is the one that most developers - will use, and the one that receives the most attention from the core - contributors. If you decide to go with a different kernel and hit a bug, - please try to reproduce it with the official kernels first. +The officially supported kernel is the one recommended by the +:ref:`ubuntu_linux` installation path. It is the one that most developers +will use, and the one that receives the most attention from the core +contributors. If you decide to go with a different kernel and hit a bug, +please try to reproduce it with the official kernels first. If you cannot or do not want to use the "official" kernels, here is some technical background about the features (both optional and From ac12d9b409aec16a0cfbade7ed576e6bf8832b82 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 16:18:54 +0000 Subject: [PATCH 50/54] Add no cache for docker build Add a new flag to disable the image cache when building images. Upstream-commit: 3a123bc479457c4dfa14e39b7c42d9a9dccf8c32 Component: engine --- components/engine/api.go | 7 ++- components/engine/buildfile.go | 55 +++++++++++-------- components/engine/buildfile_test.go | 2 +- components/engine/commands.go | 4 ++ .../sources/api/docker_remote_api_v1.4.rst | 1 + .../sources/commandline/command/build.rst | 1 + 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/components/engine/api.go b/components/engine/api.go index b8b7897c32..95b9c98de8 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -793,6 +793,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") + rawNoCache := r.FormValue("nocache") repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader @@ -839,8 +840,12 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if err != nil { return err } + noCache, err := getBoolParam(rawNoCache) + if err != nil { + return err + } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache) id, err := b.Build(context) if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 736725e915..159d7ba704 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -26,11 +26,12 @@ type buildFile struct { builder *Builder srv *Server - image string - maintainer string - config *Config - context string - verbose bool + image string + maintainer string + config *Config + context string + verbose bool + utilizeCache bool tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -94,15 +95,17 @@ func (b *buildFile) CmdRun(args string) error { utils.Debugf("Command to be executed: %v", b.config.Cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } cid, err := b.run() @@ -397,16 +400,19 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } + container, err := b.builder.Create(b.config) if err != nil { return err @@ -500,7 +506,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache bool) BuildFile { return &buildFile{ builder: NewBuilder(srv.runtime), runtime: srv.runtime, @@ -510,5 +516,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, + utilizeCache: utilizeCache, } } diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index 78e53b8419..eda047d8ea 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -227,7 +227,7 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { ip := runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false) + buildfile := NewBuildFile(srv, ioutil.Discard, false, true) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) diff --git a/components/engine/commands.go b/components/engine/commands.go index 95ddca1f1d..7a1935ee61 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -160,6 +160,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") suppressOutput := cmd.Bool("q", false, "Suppress verbose build output") + noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image") if err := cmd.Parse(args); err != nil { return nil @@ -208,6 +209,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if isRemote { v.Set("remote", cmd.Arg(0)) } + if *noCache { + v.Set("nocache", "1") + } req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index 6ee0b35fa2..6830bacde0 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -928,6 +928,7 @@ Build an image from Dockerfile via stdin :query t: tag to be applied to the resulting image in case of success :query q: suppress verbose build output + :query nocache: do not use the cache when building the image :statuscode 200: no error :statuscode 500: server error diff --git a/components/engine/docs/sources/commandline/command/build.rst b/components/engine/docs/sources/commandline/command/build.rst index 45b6d2ec8e..fcd78dd1ab 100644 --- a/components/engine/docs/sources/commandline/command/build.rst +++ b/components/engine/docs/sources/commandline/command/build.rst @@ -12,6 +12,7 @@ Build a new container image from the source code at PATH -t="": Tag to be applied to the resulting image in case of success. -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context From b7d9cc36c6037696ffb6d7f0a495f9150123bec6 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 2 Aug 2013 19:12:38 +0000 Subject: [PATCH 51/54] Add unit tests for build no cache Upstream-commit: b9f06959244e3f77eb212c7c234b06eb7b750999 Component: engine --- components/engine/buildfile_test.go | 102 ++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 19 deletions(-) diff --git a/components/engine/buildfile_test.go b/components/engine/buildfile_test.go index eda047d8ea..0e2d9ecefc 100644 --- a/components/engine/buildfile_test.go +++ b/components/engine/buildfile_test.go @@ -195,21 +195,23 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t) + buildImage(ctx, t, nil, true) } } -func buildImage(context testContextTemplate, t *testing.T) *Image { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image { + if srv == nil { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), + srv = &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } } httpServer, err := mkTestingFileServer(context.remoteFiles) @@ -224,10 +226,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { } port := httpServer.URL[idx+1:] - ip := runtime.networkManager.bridgeNetwork.IP + ip := srv.runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false, true) + buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) @@ -245,7 +247,7 @@ func TestVolume(t *testing.T) { from {IMAGE} volume /test cmd Hello world - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if len(img.Config.Volumes) == 0 { t.Fail() @@ -261,7 +263,7 @@ func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if img.Author != "dockerio" { t.Fail() @@ -273,7 +275,7 @@ func TestBuildEnv(t *testing.T) { from {IMAGE} env port 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) hasEnv := false for _, envVar := range img.Config.Env { if envVar == "port=4243" { @@ -291,7 +293,7 @@ func TestBuildCmd(t *testing.T) { from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -308,7 +310,7 @@ func TestBuildExpose(t *testing.T) { from {IMAGE} expose 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -320,8 +322,70 @@ func TestBuildEntrypoint(t *testing.T) { from {IMAGE} entrypoint ["/bin/echo"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Entrypoint[0] != "/bin/echo" { } } + +func TestBuildImageWithCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, true) + + if imageId != img.ID { + t.Logf("Image ids should match: %s != %s", imageId, img.ID) + t.Fail() + } +} + +func TestBuildImageWithoutCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, false) + + if imageId == img.ID { + t.Logf("Image ids should not match: %s == %s", imageId, img.ID) + t.Fail() + } +} From 48643d0043c3ed12e37db9aa4f4ea315edf889b8 Mon Sep 17 00:00:00 2001 From: Jonathan Rudenberg Date: Fri, 2 Aug 2013 19:18:02 -0300 Subject: [PATCH 52/54] Revert "Bind daemon to 0.0.0.0 in Vagrant. Fixes #1304" This reverts commit bdc79ac8b2dfad302f9e144711067a566726cfa2. Upstream-commit: 07fee445593982ef9063a184c05ba93ecacd2e65 Component: engine --- components/engine/Vagrantfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index 7258af5bf7..aadabb8711 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -20,8 +20,6 @@ Vagrant::Config.run do |config| pkg_cmd = "apt-get update -qq; apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/lxc-docker; apt-get update -qq; " \ "apt-get install -q -y lxc-docker; " - # Listen on all interfaces so that the daemon is accessible from the host - pkg_cmd << "sed -i -E 's| /usr/bin/docker -d| /usr/bin/docker -d -H 0.0.0.0|' /etc/init/docker.conf;" # Add X.org Ubuntu backported 3.8 kernel pkg_cmd << "add-apt-repository -y ppa:ubuntu-x-swat/r-lts-backport; " \ "apt-get update -qq; apt-get install -q -y linux-image-3.8.0-19-generic; " From 46b9c958878fd716ab26af38c3192dffe1c5cc1b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 2 Aug 2013 15:58:10 -0700 Subject: [PATCH 53/54] Fix TestEnv Upstream-commit: dde8f74ceae83f26386ec29e42f615fdc7945e80 Component: engine --- components/engine/container_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/container_test.go b/components/engine/container_test.go index a1ac0bd33a..f29ae9e4ea 100644 --- a/components/engine/container_test.go +++ b/components/engine/container_test.go @@ -960,6 +960,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", + "HOSTNAME=" + container.ShortID(), } sort.Strings(goodEnv) if len(goodEnv) != len(actualEnv) { From 3029f070e629c91386d2bb05829f925f1749aff3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 30 Jul 2013 22:48:20 +0000 Subject: [PATCH 54/54] Return JSONError for HTTPResponse error Upstream-commit: dae585c6e4c19817b2dbd106171728a0bb564ccc Component: engine --- components/engine/api.go | 12 ++++-------- components/engine/commands.go | 2 +- components/engine/registry/registry.go | 4 ++-- components/engine/utils/error.go | 18 ------------------ components/engine/utils/utils.go | 13 ++++++++++++- components/engine/utils_test.go | 2 +- 6 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 components/engine/utils/error.go diff --git a/components/engine/api.go b/components/engine/api.go index 5869669df0..4ad2ba461a 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -388,7 +388,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht if image != "" { //pull if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -396,7 +396,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } else { //import if err := srv.ImageImport(src, repo, tag, r.Body, w, sf); err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } return err @@ -441,7 +441,7 @@ func postImagesInsert(srv *Server, version float64, w http.ResponseWriter, r *ht imgID, err := srv.ImageInsert(name, url, path, w, sf) if err != nil { if sf.Used() { - w.Write(sf.FormatError(err, 0)) + w.Write(sf.FormatError(err)) return nil } } @@ -472,11 +472,7 @@ func postImagesPush(srv *Server, version float64, w http.ResponseWriter, r *http sf := utils.NewStreamFormatter(version > 1.0) if err := srv.ImagePush(name, w, sf, authConfig); err != nil { if sf.Used() { - var code int - if httpErr, ok := err.(*utils.HTTPRequestError); ok { - code = httpErr.StatusCode - } - w.Write(sf.FormatError(err, code)) + w.Write(sf.FormatError(err)) return nil } return err diff --git a/components/engine/commands.go b/components/engine/commands.go index 9ad2c367ae..355e91f4bf 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -31,7 +31,7 @@ const VERSION = "0.5.0-dev" var ( GITCOMMIT string - AuthRequiredError error = fmt.Errorf("Authentication is required.") + AuthRequiredError = fmt.Errorf("Authentication is required.") ) func (cli *DockerCli) getMethod(name string) (reflect.Method, bool) { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index ed6f4c7df8..5b8480d183 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -427,9 +427,9 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } return tarsumLayer.Sum(jsonRaw), nil } diff --git a/components/engine/utils/error.go b/components/engine/utils/error.go deleted file mode 100644 index 7e3c846ebc..0000000000 --- a/components/engine/utils/error.go +++ /dev/null @@ -1,18 +0,0 @@ -package utils - -import ( - "net/http" -) - -type HTTPRequestError struct { - Message string - StatusCode int -} - -func (e *HTTPRequestError) Error() string { - return e.Message -} - -func NewHTTPRequestError(msg string, resp *http.Response) error { - return &HTTPRequestError{Message: msg, StatusCode: resp.StatusCode} -} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 659d960d0a..2323829f65 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -625,6 +625,13 @@ func (e *JSONError) Error() string { return e.Message } +func NewHTTPRequestError(msg string, res *http.Response) error { + return &JSONError{ + Message: msg, + Code: res.StatusCode, + } +} + func (jm *JSONMessage) Display(out io.Writer) error { if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) @@ -666,7 +673,11 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: &JSONError{Code: code, Message: err.Error()}, ErrorMessage: err.Error()}); err == nil { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") diff --git a/components/engine/utils_test.go b/components/engine/utils_test.go index c4adeb4a74..91df6183fc 100644 --- a/components/engine/utils_test.go +++ b/components/engine/utils_test.go @@ -191,7 +191,7 @@ func TestMergeConfig(t *testing.T) { if len(configUser.Volumes) != 3 { t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes)) } - for v, _ := range configUser.Volumes { + for v := range configUser.Volumes { if v != "/test1" && v != "/test2" && v != "/test3" { t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v) }