From 6d7f428b7a407f5a483072bf827c86e9bf5a4eb5 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 11 Dec 2014 17:55:15 -0800 Subject: [PATCH 01/26] Adds support for v2 registry login summary of changes: registry/auth.go - More logging around the login functions - split Login() out to handle different code paths for v1 (unchanged logic) and v2 (does not currently do account creation) - handling for either basic or token based login attempts registry/authchallenge.go - New File - credit to Brian Bland (github: BrianBland) - handles parsing of WWW-Authenticate response headers registry/endpoint.go - EVEN MOAR LOGGING - Many edits throught to make the coad less dense. Sparse code is more readable code. - slit Ping() out to handle different code paths for v1 (unchanged logic) and v2. - Updated Endpoint struct type to include an entry for authorization challenges discovered during ping of a v2 registry. - If registry endpoint version is unknown, v2 code path is first attempted, then fallback to v1 upon failure. registry/service.go - STILL MOAR LOGGING - simplified the logic around starting the 'auth' job. registry/session.go - updated use of a registry.Endpoint struct field. registry/token.go - New File - Handles getting token from the parameters of a token auth challenge. - Modified from function written by Brian Bland (see above credit). registry/types.go - Removed 'DefaultAPIVersion' in lieu of 'APIVersionUnknown = 0'` Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- registry/auth.go | 114 ++++++++++++++++++++++++++- registry/authchallenge.go | 150 ++++++++++++++++++++++++++++++++++++ registry/endpoint.go | 158 ++++++++++++++++++++++++++++---------- registry/endpoint_test.go | 6 +- registry/service.go | 46 +++++++---- registry/session.go | 2 +- registry/token.go | 70 +++++++++++++++++ registry/types.go | 5 +- 8 files changed, 486 insertions(+), 65 deletions(-) create mode 100644 registry/authchallenge.go create mode 100644 registry/token.go diff --git a/registry/auth.go b/registry/auth.go index 102078d7a2dd9..2044236cfb923 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -11,6 +11,7 @@ import ( "path" "strings" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -144,8 +145,18 @@ func SaveConfig(configFile *ConfigFile) error { return nil } -// try to register/login to the registry server -func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { +// Login tries to register/login to the registry server. +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + // Separates the v2 registry login logic from the v1 logic. + if registryEndpoint.Version == APIVersion2 { + return loginV2(authConfig, registryEndpoint, factory) + } + + return loginV1(authConfig, registryEndpoint, factory) +} + +// loginV1 tries to register/login to the v1 registry server. +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( status string reqBody []byte @@ -161,6 +172,8 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e serverAddress = authConfig.ServerAddress ) + log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") } @@ -253,6 +266,103 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return status, nil } +// loginV2 tries to login to the v2 registry server. The given registry endpoint has been +// pinged or setup with a list of authorization challenges. Each of these challenges are +// tried until one of them succeeds. Currently supported challenge schemes are: +// HTTP Basic Authorization +// Token Authorization with a separate token issuing server +// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For +// now, users should create their account through other means like directly from a web page +// served by the v2 registry service provider. Whether this will be supported in the future +// is to be determined. +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + + var ( + err error + allErrors []error + ) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + case "bearer": + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + default: + // Unsupported challenge types are explicitly skipped. + err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) + } + + if err == nil { + return "Login Succeeded", nil + } + + log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + + allErrors = append(allErrors, err) + } + + return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +} + +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return err + } + + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + // this method matches a auth configuration to a server address or a url func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() diff --git a/registry/authchallenge.go b/registry/authchallenge.go new file mode 100644 index 0000000000000..e300d82a0580e --- /dev/null +++ b/registry/authchallenge.go @@ -0,0 +1,150 @@ +package registry + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []*AuthorizationChallenge { + var challenges []*AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/registry/endpoint.go b/registry/endpoint.go index 95680c5efca39..5c5b0520001b2 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -15,28 +15,31 @@ import ( // for mocking in unit tests var lookupIP = net.LookupIP -// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForAPIVersion(hostname string) (string, APIVersion) { +// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. +func scanForAPIVersion(address string) (string, APIVersion) { var ( chunks []string apiVersionStr string ) - if strings.HasSuffix(hostname, "/") { - chunks = strings.Split(hostname[:len(hostname)-1], "/") - apiVersionStr = chunks[len(chunks)-1] - } else { - chunks = strings.Split(hostname, "/") - apiVersionStr = chunks[len(chunks)-1] + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + for k, v := range apiVersions { if apiVersionStr == v { - hostname = strings.Join(chunks[:len(chunks)-1], "/") - return hostname, k + address = strings.Join(chunks[:len(chunks)-1], "/") + return address, k } } - return hostname, DefaultAPIVersion + + return address, APIVersionUnknown } +// NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) @@ -44,81 +47,124 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { return nil, err } + log.Debugf("pinging registry endpoint %s", endpoint) + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - - //TODO: triggering highland build can be done there without "failing" - if index.Secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" - _, err2 := endpoint.Ping() - if err2 == nil { + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { return endpoint, nil } - return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { + +func newEndpoint(address string, secure bool) (*Endpoint, error) { var ( - endpoint = Endpoint{} - trimmedHostname string - err error + endpoint = new(Endpoint) + trimmedAddress string + err error ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname + + if !strings.HasPrefix(address, "http") { + address = "https://" + address } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) - if err != nil { + + trimmedAddress, endpoint.Version = scanForAPIVersion(address) + + if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.secure = secure - return &endpoint, nil + endpoint.IsSecure = secure + return endpoint, nil } func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { return NewEndpoint(repoInfo.Index) } +// Endpoint stores basic information about a registry endpoint. type Endpoint struct { - URL *url.URL - Version APIVersion - secure bool + URL *url.URL + Version APIVersion + IsSecure bool + AuthChallenges []*AuthorizationChallenge } // Get the formated URL for the root of this registry Endpoint -func (e Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +func (e *Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL, e.Version) +} + +// VersionString returns a formatted string of this +// endpoint address using the given API Version. +func (e *Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL, version) } -func (e Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *Endpoint) Path(path string) string { + return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) } -func (e Endpoint) Ping() (RegistryInfo, error) { +func (e *Endpoint) Ping() (RegistryInfo, error) { + // The ping logic to use is determined by the registry endpoint version. + switch e.Version { + case APIVersion1: + return e.pingV1() + case APIVersion2: + return e.pingV2() + } + + // APIVersionUnknown + // We should try v2 first... + e.Version = APIVersion2 + regInfo, errV2 := e.pingV2() + if errV2 == nil { + return regInfo, nil + } + + // ... then fallback to v1. + e.Version = APIVersion1 + regInfo, errV1 := e.pingV1() + if errV1 == nil { + return regInfo, nil + } + + e.Version = APIVersionUnknown + return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) +} + +func (e *Endpoint) pingV1() (RegistryInfo, error) { + log.Debugf("attempting v1 ping for registry endpoint %s", e) + if e.String() == IndexServerAddress() { - // Skip the check, we now this one is valid + // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.String()+"_ping", nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,7 +173,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier @@ -157,3 +203,33 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +func (e *Endpoint) pingV2() (RegistryInfo, error) { + log.Debugf("attempting v2 ping for registry endpoint %s", e) + + req, err := http.NewRequest("GET", e.Path(""), nil) + if err != nil { + return RegistryInfo{}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + if err != nil { + return RegistryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + // It would seem that no authentication/authorization is required. + // So we don't need to parse/add any authorization schemes. + return RegistryInfo{Standalone: true}, nil + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + e.AuthChallenges = parseAuthHeader(resp.Header) + return RegistryInfo{}, nil + } + + return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) +} diff --git a/registry/endpoint_test.go b/registry/endpoint_test.go index b691a4fb98595..f6489034feb7b 100644 --- a/registry/endpoint_test.go +++ b/registry/endpoint_test.go @@ -8,8 +8,10 @@ func TestEndpointParse(t *testing.T) { expected string }{ {IndexServerAddress(), IndexServerAddress()}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { e, err := newEndpoint(td.str, false) diff --git a/registry/service.go b/registry/service.go index c34e384236ce7..048340224869b 100644 --- a/registry/service.go +++ b/registry/service.go @@ -1,6 +1,7 @@ package registry import ( + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,28 +39,39 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var authConfig = new(AuthConfig) + var ( + authConfig = new(AuthConfig) + endpoint *Endpoint + index *IndexInfo + status string + err error + ) job.GetenvJson("authConfig", authConfig) - if authConfig.ServerAddress != "" { - index, err := ResolveIndexInfo(job, authConfig.ServerAddress) - if err != nil { - return job.Error(err) - } - if !index.Official { - endpoint, err := NewEndpoint(index) - if err != nil { - return job.Error(err) - } - authConfig.ServerAddress = endpoint.String() - } - } - - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + addr := authConfig.ServerAddress + if addr == "" { + // Use the official registry address if not specified. + addr = IndexServerAddress() + } + + if index, err = ResolveIndexInfo(job, addr); err != nil { return job.Error(err) } + + if endpoint, err = NewEndpoint(index); err != nil { + log.Errorf("unable to get new registry endpoint: %s", err) + return job.Error(err) + } + + authConfig.ServerAddress = endpoint.String() + + if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { + log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + return job.Error(err) + } + + log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return engine.StatusOK diff --git a/registry/session.go b/registry/session.go index 28cf18fbe38c8..43aa9283010c2 100644 --- a/registry/session.go +++ b/registry/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) } // Retrieve the history of a given image from the Registry. diff --git a/registry/token.go b/registry/token.go new file mode 100644 index 0000000000000..0403734f87d6b --- /dev/null +++ b/registry/token.go @@ -0,0 +1,70 @@ +package registry + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/utils" +) + +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + if realmURL.Scheme == "" { + if registryEndpoint.IsSecure { + realmURL.Scheme = "https" + } else { + realmURL.Scheme = "http" + } + } + + req, err := factory.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + reqParams.Add("account", username) + + req.URL.RawQuery = reqParams.Encode() + req.SetBasicAuth(username, password) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + token = resp.Header.Get("X-Auth-Token") + if token == "" { + return "", errors.New("token server did not include a token in the response header") + } + + return token, nil +} diff --git a/registry/types.go b/registry/types.go index fbbc0e7098a66..bd0bf8b75b0fd 100644 --- a/registry/types.go +++ b/registry/types.go @@ -55,14 +55,15 @@ func (av APIVersion) String() string { return apiVersions[av] } -var DefaultAPIVersion APIVersion = APIVersion1 var apiVersions = map[APIVersion]string{ 1: "v1", 2: "v2", } +// API Version identifiers. const ( - APIVersion1 = iota + 1 + APIVersionUnknown = iota + APIVersion1 APIVersion2 ) From 964763bafc554c4d9ccfdd99faadc8499d93af9a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 22 Oct 2014 11:07:03 -0700 Subject: [PATCH 02/26] Add trust key creation on client Signed-off-by: Derek McGowan (github: dmcgowan) --- docker/docker.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 3137f5c99fc46..84ffeace9a990 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "strings" log "github.com/Sirupsen/logrus" @@ -15,6 +16,7 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const ( @@ -77,6 +79,23 @@ func main() { } protoAddrParts := strings.SplitN(flHosts[0], "://", 2) + err := os.MkdirAll(path.Dir(*flTrustKey), 0700) + if err != nil { + log.Fatal(err) + } + trustKey, err := libtrust.LoadKeyFile(*flTrustKey) + if err == libtrust.ErrKeyFileDoesNotExist { + trustKey, err = libtrust.GenerateECP256PrivateKey() + if err != nil { + log.Fatalf("Error generating key: %s", err) + } + if err := libtrust.SaveKey(*flTrustKey, trustKey); err != nil { + log.Fatalf("Error saving key file: %s", err) + } + } else if err != nil { + log.Fatalf("Error loading key file: %s", err) + } + var ( cli *client.DockerCli tlsConfig tls.Config @@ -118,9 +137,9 @@ func main() { } if *flTls || *flTlsVerify { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], &tlsConfig) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], &tlsConfig) } else { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], nil) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], nil) } if err := cli.Cmd(flag.Args()...); err != nil { From b94732e527562f6a392e75643ae2beb64360de20 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 30 Sep 2014 17:03:57 -0700 Subject: [PATCH 03/26] Push flow Signed-off-by: Derek McGowan (github: dmcgowan) --- api/client/commands.go | 23 +++++++- api/server/server.go | 19 +++++++ graph/manifest.go | 116 +++++++++++++++++++++++++++++++++++++++++ graph/push.go | 90 ++++++++++++++++++++++++++++++++ graph/service.go | 1 + registry/session_v2.go | 7 ++- 6 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 graph/manifest.go diff --git a/api/client/commands.go b/api/client/commands.go index d6e2c94f3d87b..5ea237f466bc7 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -43,6 +43,7 @@ import ( "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const ( @@ -1195,6 +1196,26 @@ func (cli *DockerCli) CmdPush(args ...string) error { v := url.Values{} v.Set("tag", tag) + + body, _, err := readBody(cli.call("GET", "/images/"+remote+"/manifest?"+v.Encode(), nil, false)) + if err != nil { + return err + } + + js, err := libtrust.NewJSONSignature(body) + if err != nil { + return err + } + err = js.Sign(cli.key) + if err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + push := func(authConfig registry.AuthConfig) error { buf, err := json.Marshal(authConfig) if err != nil { @@ -1204,7 +1225,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { base64.URLEncoding.EncodeToString(buf), } - return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, map[string][]string{ + return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), bytes.NewReader(signedBody), cli.out, map[string][]string{ "X-Registry-Auth": registryAuthHeader, }) } diff --git a/api/server/server.go b/api/server/server.go index cfaa5f43ab131..18d2657fdf0ed 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -608,6 +608,18 @@ func getImagesSearch(eng *engine.Engine, version version.Version, w http.Respons return job.Run() } +func getImageManifest(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := parseForm(r); err != nil { + return err + } + + job := eng.Job("image_manifest", vars["name"]) + job.Setenv("tag", r.Form.Get("tag")) + job.Stdout.Add(utils.NewWriteFlusher(w)) + + return job.Run() +} + func postImagesPush(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") @@ -639,9 +651,15 @@ func postImagesPush(eng *engine.Engine, version version.Version, w http.Response } } + manifest, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + job := eng.Job("push", vars["name"]) job.SetenvJson("metaHeaders", metaHeaders) job.SetenvJson("authConfig", authConfig) + job.Setenv("manifest", string(manifest)) job.Setenv("tag", r.Form.Get("tag")) if version.GreaterThan("1.0") { job.SetenvBool("json", true) @@ -1276,6 +1294,7 @@ func createRouter(eng *engine.Engine, logging, enableCors bool, dockerVersion st "/images/viz": getImagesViz, "/images/search": getImagesSearch, "/images/get": getImagesGet, + "/images/{name:.*}/manifest": getImageManifest, "/images/{name:.*}/get": getImagesGet, "/images/{name:.*}/history": getImagesHistory, "/images/{name:.*}/json": getImagesByName, diff --git a/graph/manifest.go b/graph/manifest.go new file mode 100644 index 0000000000000..39cabb6e41ca5 --- /dev/null +++ b/graph/manifest.go @@ -0,0 +1,116 @@ +package graph + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "path" + + "github.com/docker/docker/engine" + "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/registry" + "github.com/docker/docker/runconfig" +) + +func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("usage: %s NAME", job.Name) + } + name := job.Args[0] + tag := job.Getenv("tag") + if tag == "" { + tag = "latest" + } + + // Resolve the Repository name from fqn to endpoint + name + _, remoteName, err := registry.ResolveRepositoryName(name) + if err != nil { + return job.Error(err) + } + + manifest := ®istry.ManifestData{ + Name: remoteName, + Tag: tag, + SchemaVersion: 1, + } + localRepo, exists := s.Repositories[name] + if !exists { + return job.Errorf("Repo does not exist: %s", name) + } + + layerId, exists := localRepo[tag] + if !exists { + return job.Errorf("Tag does not exist for %s: %s", name, tag) + } + tarsums := make([]string, 0, 4) + layersSeen := make(map[string]bool) + + layer, err := s.graph.Get(layerId) + if err != nil { + return job.Error(err) + } + if layer.Config == nil { + return job.Errorf("Missing layer configuration") + } + manifest.Architecture = layer.Architecture + var metadata runconfig.Config + metadata = *layer.Config + history := make([]string, 0, cap(tarsums)) + + for ; layer != nil; layer, err = layer.GetParent() { + if err != nil { + return job.Error(err) + } + + if layersSeen[layer.ID] { + break + } + if layer.Config != nil && metadata.Image != layer.ID { + err = runconfig.Merge(&metadata, layer.Config) + if err != nil { + return job.Error(err) + } + } + + archive, err := layer.TarLayer() + if err != nil { + return job.Error(err) + } + + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version0) + if err != nil { + return job.Error(err) + } + if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { + return job.Error(err) + } + + tarId := tarSum.Sum(nil) + // Save tarsum to image json + + tarsums = append(tarsums, tarId) + + layersSeen[layer.ID] = true + jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) + if err != nil { + return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) + } + history = append(history, string(jsonData)) + } + + manifest.BlobSums = tarsums + manifest.History = history + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return job.Error(err) + } + + _, err = job.Stdout.Write(manifestBytes) + if err != nil { + return job.Error(err) + } + + return engine.StatusOK +} diff --git a/graph/push.go b/graph/push.go index 09e13a5cff615..b99b7197eaae1 100644 --- a/graph/push.go +++ b/graph/push.go @@ -1,14 +1,17 @@ package graph import ( + "bytes" "fmt" "io" "io/ioutil" "os" "path" + "strings" log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" + "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/registry" "github.com/docker/docker/utils" @@ -206,6 +209,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } tag := job.Getenv("tag") + manifestBytes := job.Getenv("manifest") job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) @@ -225,6 +229,92 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err2) } + var isOfficial bool + if endpoint.String() == registry.IndexServerAddress() { + isOfficial = isOfficialName(remoteName) + if isOfficial && strings.IndexRune(remoteName, '/') == -1 { + remoteName = "library/" + remoteName + } + } + + if len(tag) == 0 { + tag = DEFAULTTAG + } + if isOfficial || endpoint.Version == registry.APIVersion2 { + j := job.Eng.Job("trust_update_base") + if err = j.Run(); err != nil { + return job.Errorf("error updating trust base graph: %s", err) + } + + repoData, err := r.PushImageJSONIndex(remoteName, []*registry.ImgData{}, false, nil) + if err != nil { + return job.Error(err) + } + + // try via manifest + manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) + if err != nil { + return job.Errorf("error verifying manifest: %s", err) + } + + if len(manifest.FSLayers) != len(manifest.History) { + return job.Errorf("length of history not equal to number of layers") + } + + if !verified { + log.Debugf("Pushing unverified image") + } + + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { + var ( + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) + ) + + sumParts := strings.SplitN(sumStr, ":", 2) + if len(sumParts) < 2 { + return job.Errorf("Invalid checksum: %s", sumStr) + } + manifestSum := sumParts[1] + + // for each layer, check if it exists ... + // XXX wait this requires having the TarSum of the layer.tar first + // skip this step for now. Just push the layer every time for this naive implementation + //shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string) + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return job.Errorf("Failed to parse json: %s", err) + } + + img, err = s.graph.Get(img.ID) + if err != nil { + return job.Error(err) + } + + arch, err := img.TarLayer() + if err != nil { + return job.Errorf("Could not get tar layer: %s", err) + } + + _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + if err != nil { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return job.Error(err) + } + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } + + // push the manifest + err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader([]byte(manifestBytes)), repoData.Tokens) + if err != nil { + return job.Error(err) + } + + // done, no fallback to V1 + return engine.StatusOK + } + if err != nil { reposLen := 1 if tag == "" { diff --git a/graph/service.go b/graph/service.go index 2858d9b3e6b91..675e12a1a9537 100644 --- a/graph/service.go +++ b/graph/service.go @@ -25,6 +25,7 @@ func (s *TagStore) Install(eng *engine.Engine) error { "import": s.CmdImport, "pull": s.CmdPull, "push": s.CmdPush, + "image_manifest": s.CmdManifest, } { if err := eng.Register(name, handler); err != nil { return fmt.Errorf("Could not register %q: %v", name, err) diff --git a/registry/session_v2.go b/registry/session_v2.go index 20e9e2ee9cac4..0498bf702e90a 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -267,7 +267,7 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { vars := map[string]string{ "imagename": imageName, "sumtype": sumType, @@ -285,6 +285,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", err } setTokenAuth(req, token) + req.Header.Set("X-Tarsum", sumStr) res, _, err := r.doRequest(req) if err != nil { return "", err @@ -309,6 +310,10 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) } + if sumInfo.Checksum != sumStr { + return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) + } + // XXX this is a json struct from the registry, with its checksum return sumInfo.Checksum, nil } From a3ed28956ad24bb088bb943e2bd76d23287b37bd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:34:52 -0700 Subject: [PATCH 04/26] Update manifest format for push Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/manifest.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/graph/manifest.go b/graph/manifest.go index 39cabb6e41ca5..752b782308339 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -43,7 +43,6 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { if !exists { return job.Errorf("Tag does not exist for %s: %s", name, tag) } - tarsums := make([]string, 0, 4) layersSeen := make(map[string]bool) layer, err := s.graph.Get(layerId) @@ -54,9 +53,10 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { return job.Errorf("Missing layer configuration") } manifest.Architecture = layer.Architecture + manifest.FSLayers = make([]*registry.FSLayer, 0, 4) + manifest.History = make([]*registry.ManifestHistory, 0, 4) var metadata runconfig.Config metadata = *layer.Config - history := make([]string, 0, cap(tarsums)) for ; layer != nil; layer, err = layer.GetParent() { if err != nil { @@ -89,19 +89,16 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { tarId := tarSum.Sum(nil) // Save tarsum to image json - tarsums = append(tarsums, tarId) + manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) layersSeen[layer.ID] = true jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) if err != nil { return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) } - history = append(history, string(jsonData)) + manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) } - manifest.BlobSums = tarsums - manifest.History = history - manifestBytes, err := json.MarshalIndent(manifest, "", " ") if err != nil { return job.Error(err) From 83e4eb99fea7ec1a98826e99660b28e4da11ad39 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:32:16 -0700 Subject: [PATCH 05/26] Use tarsum dev version to fix mtime issue Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/manifest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/manifest.go b/graph/manifest.go index 752b782308339..ddcb22b650cce 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -78,7 +78,7 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { return job.Error(err) } - tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version0) + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.VersionDev) if err != nil { return job.Error(err) } From fa006d694d05481e482e4208b55a62ce17ce73ed Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 14 Nov 2014 16:22:06 -0800 Subject: [PATCH 06/26] Update push to use mount blob endpoint Using mount blob prevents repushing images which have already been uploaded Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/push.go | 14 ++++++++++++-- registry/session_v2.go | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/graph/push.go b/graph/push.go index b99b7197eaae1..653b0eccb5843 100644 --- a/graph/push.go +++ b/graph/push.go @@ -297,12 +297,22 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Errorf("Could not get tar layer: %s", err) } - _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + // Call mount blob + exists, err := r.PostV2ImageMountBlob(remoteName, sumParts[0], manifestSum, repoData.Tokens) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) } - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + if !exists { + _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + if err != nil { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return job.Error(err) + } + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } else { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) + } } // push the manifest diff --git a/registry/session_v2.go b/registry/session_v2.go index 0498bf702e90a..86d0c228a772b 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -34,7 +34,7 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } @@ -184,7 +184,7 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s case 200: // return something indicating no push needed return true, nil - case 300: + case 404: // return something indicating blob push needed return false, nil } From 914be9752d949a5dad3a01db227e86fe31b009ed Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 12 Dec 2014 13:30:12 -0800 Subject: [PATCH 07/26] Update token response handling Registry authorization token is now taken from the response body rather than the repsonse header. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- registry/token.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/registry/token.go b/registry/token.go index 0403734f87d6b..2504863048918 100644 --- a/registry/token.go +++ b/registry/token.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "errors" "fmt" "net/http" @@ -10,6 +11,10 @@ import ( "github.com/docker/docker/utils" ) +type tokenResponse struct { + Token string `json:"token"` +} + func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { realm, ok := params["realm"] if !ok { @@ -57,14 +62,20 @@ func getToken(username, password string, params map[string]string, registryEndpo } defer resp.Body.Close() - if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) } - token = resp.Header.Get("X-Auth-Token") - if token == "" { - return "", errors.New("token server did not include a token in the response header") + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) + } + + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") } - return token, nil + return tr.Token, nil } From 0c6610952a11a8d53ee88496dae5be2719832e1b Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 12 Dec 2014 11:27:22 -0800 Subject: [PATCH 08/26] Registry V2 HTTP route and error code definitions This package, ported from next-generation docker regsitry, includes route and error definitions. These facilitate compliant V2 client implementation. The portions of the HTTP API that are included in this package are considered to be locked down and should only be changed through a careful change proposal. Descriptor definitions package layout may change without affecting API behavior until the exported Go API is ready to be locked down. When the new registry stabilizes and becomes the master branch, this package can be vendored from the registry. Signed-off-by: Stephen J Day --- registry/v2/descriptors.go | 144 +++++++++++++++++++++++++++++ registry/v2/doc.go | 13 +++ registry/v2/errors.go | 185 +++++++++++++++++++++++++++++++++++++ registry/v2/errors_test.go | 165 +++++++++++++++++++++++++++++++++ registry/v2/routes.go | 69 ++++++++++++++ registry/v2/routes_test.go | 184 ++++++++++++++++++++++++++++++++++++ registry/v2/urls.go | 165 +++++++++++++++++++++++++++++++++ registry/v2/urls_test.go | 100 ++++++++++++++++++++ 8 files changed, 1025 insertions(+) create mode 100644 registry/v2/descriptors.go create mode 100644 registry/v2/doc.go create mode 100644 registry/v2/errors.go create mode 100644 registry/v2/errors_test.go create mode 100644 registry/v2/routes.go create mode 100644 registry/v2/routes_test.go create mode 100644 registry/v2/urls.go create mode 100644 registry/v2/urls_test.go diff --git a/registry/v2/descriptors.go b/registry/v2/descriptors.go new file mode 100644 index 0000000000000..68d182411d9c1 --- /dev/null +++ b/registry/v2/descriptors.go @@ -0,0 +1,144 @@ +package v2 + +import "net/http" + +// TODO(stevvooe): Add route descriptors for each named route, along with +// accepted methods, parameters, returned status codes and error codes. + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCodes provides a list of status under which this error + // condition may arise. If it is empty, the error condition may be seen + // for any status code. + HTTPStatusCodes []int +} + +// ErrorDescriptors provides a list of HTTP API Error codes that may be +// encountered when interacting with the registry API. +var ErrorDescriptors = []ErrorDescriptor{ + { + Code: ErrorCodeUnknown, + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + }, + { + Code: ErrorCodeDigestInvalid, + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeSizeInvalid, + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeNameInvalid, + Value: "NAME_INVALID", + Message: "manifest name did not match URI", + Description: `During a manifest upload, if the name in the manifest + does not match the uri name, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeTagInvalid, + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeNameUnknown, + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestUnknown, + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestInvalid, + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeManifestUnverified, + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeBlobUnknown, + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + + { + Code: ErrorCodeBlobUploadUnknown, + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, +} + +var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor +var idToDescriptors map[string]ErrorDescriptor + +func init() { + errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) + idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) + + for _, descriptor := range ErrorDescriptors { + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + } +} diff --git a/registry/v2/doc.go b/registry/v2/doc.go new file mode 100644 index 0000000000000..30fe2271a19b5 --- /dev/null +++ b/registry/v2/doc.go @@ -0,0 +1,13 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal. +// +// Currently, while the HTTP API definitions are considered stable, the Go API +// exports are considered unstable. Go API consumers should take care when +// relying on these definitions until this message is deleted. +package v2 diff --git a/registry/v2/errors.go b/registry/v2/errors.go new file mode 100644 index 0000000000000..8c85d3a97f164 --- /dev/null +++ b/registry/v2/errors.go @@ -0,0 +1,185 @@ +package v2 + +import ( + "fmt" + "strings" +) + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +const ( + // ErrorCodeUnknown is a catch-all for errors not defined below. + ErrorCodeUnknown ErrorCode = iota + + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + // size does not match the content length. + ErrorCodeSizeInvalid + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verfication. + ErrorCodeManifestUnverified + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown +) + +// ParseErrorCode attempts to parse the error code string, returning +// ErrorCodeUnknown if the error is not known. +func ParseErrorCode(s string) ErrorCode { + desc, ok := idToDescriptors[s] + + if !ok { + return ErrorCodeUnknown + } + + return desc.Code +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", + strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), + e.Message) +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors struct { + Errors []Error `json:"errors,omitempty"` +} + +// Push pushes an error on to the error stack, with the optional detail +// argument. It is a programming error (ie panic) to push more than one +// detail at a time. +func (errs *Errors) Push(code ErrorCode, details ...interface{}) { + if len(details) > 1 { + panic("please specify zero or one detail items for this error") + } + + var detail interface{} + if len(details) > 0 { + detail = details[0] + } + + if err, ok := detail.(error); ok { + detail = err.Error() + } + + errs.PushErr(Error{ + Code: code, + Message: code.Message(), + Detail: detail, + }) +} + +// PushErr pushes an error interface onto the error stack. +func (errs *Errors) PushErr(err error) { + switch err.(type) { + case Error: + errs.Errors = append(errs.Errors, err.(Error)) + default: + errs.Errors = append(errs.Errors, Error{Message: err.Error()}) + } +} + +func (errs *Errors) Error() string { + switch errs.Len() { + case 0: + return "" + case 1: + return errs.Errors[0].Error() + default: + msg := "errors:\n" + for _, err := range errs.Errors { + msg += err.Error() + "\n" + } + return msg + } +} + +// Clear clears the errors. +func (errs *Errors) Clear() { + errs.Errors = errs.Errors[:0] +} + +// Len returns the current number of errors. +func (errs *Errors) Len() int { + return len(errs.Errors) +} diff --git a/registry/v2/errors_test.go b/registry/v2/errors_test.go new file mode 100644 index 0000000000000..d2fc091acad54 --- /dev/null +++ b/registry/v2/errors_test.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/docker/docker-registry/digest" +) + +// TestErrorCodes ensures that error code format, mappings and +// marshaling/unmarshaling. round trips are stable. +func TestErrorCodes(t *testing.T) { + for _, desc := range ErrorDescriptors { + if desc.Code.String() != desc.Value { + t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) + } + + if desc.Code.Message() != desc.Message { + t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) + } + + // Serialize the error code using the json library to ensure that we + // get a string and it works round trip. + p, err := json.Marshal(desc.Code) + + if err != nil { + t.Fatalf("error marshaling error code %v: %v", desc.Code, err) + } + + if len(p) <= 0 { + t.Fatalf("expected content in marshaled before for error code %v", desc.Code) + } + + // First, unmarshal to interface and ensure we have a string. + var ecUnspecified interface{} + if err := json.Unmarshal(p, &ecUnspecified); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if _, ok := ecUnspecified.(string); !ok { + t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) + } + + // Now, unmarshal with the error code type and ensure they are equal + var ecUnmarshaled ErrorCode + if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if ecUnmarshaled != desc.Code { + t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) + } + } +} + +// TestErrorsManagement does a quick check of the Errors type to ensure that +// members are properly pushed and marshaled. +func TestErrorsManagement(t *testing.T) { + var errs Errors + + errs.Push(ErrorCodeDigestInvalid) + errs.Push(ErrorCodeBlobUnknown, + map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + + p, err := json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } + + errs.Clear() + errs.Push(ErrorCodeUnknown) + expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" + p, err = json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } +} + +// TestMarshalUnmarshal ensures that api errors can round trip through json +// without losing information. +func TestMarshalUnmarshal(t *testing.T) { + + var errors Errors + + for _, testcase := range []struct { + description string + err Error + }{ + { + description: "unknown error", + err: Error{ + + Code: ErrorCodeUnknown, + Message: ErrorCodeUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeManifestUnknown, + Message: ErrorCodeManifestUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeBlobUnknown, + Message: ErrorCodeBlobUnknown.Descriptor().Message, + Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, + }, + }, + } { + fatalf := func(format string, args ...interface{}) { + t.Fatalf(testcase.description+": "+format, args...) + } + + unexpectedErr := func(err error) { + fatalf("unexpected error: %v", err) + } + + p, err := json.Marshal(testcase.err) + if err != nil { + unexpectedErr(err) + } + + var unmarshaled Error + if err := json.Unmarshal(p, &unmarshaled); err != nil { + unexpectedErr(err) + } + + if !reflect.DeepEqual(unmarshaled, testcase.err) { + fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) + } + + // Roll everything up into an error response envelope. + errors.PushErr(testcase.err) + } + + p, err := json.Marshal(errors) + if err != nil { + t.Fatalf("unexpected error marshaling error envelope: %v", err) + } + + var unmarshaled Errors + if err := json.Unmarshal(p, &unmarshaled); err != nil { + t.Fatalf("unexpected error unmarshaling error envelope: %v", err) + } + + if !reflect.DeepEqual(unmarshaled, errors) { + t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) + } +} diff --git a/registry/v2/routes.go b/registry/v2/routes.go new file mode 100644 index 0000000000000..7ebe61d665d46 --- /dev/null +++ b/registry/v2/routes.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "github.com/docker/docker-registry/common" + "github.com/gorilla/mux" +) + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" +) + +var allEndpoints = []string{ + RouteNameManifest, + RouteNameTags, + RouteNameBlob, + RouteNameBlobUpload, + RouteNameBlobUploadChunk, +} + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + router := mux.NewRouter(). + StrictSlash(true) + + // GET /v2/ Check Check that the registry implements API version 2(.1) + router. + Path("/v2/"). + Name(RouteNameBase) + + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Name(RouteNameManifest) + + // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Name(RouteNameTags) + + // GET /v2//blob/ Layer Fetch the blob identified by digest. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Name(RouteNameBlob) + + // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Name(RouteNameBlobUpload) + + // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. + // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. + // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Name(RouteNameBlobUploadChunk) + + return router +} diff --git a/registry/v2/routes_test.go b/registry/v2/routes_test.go new file mode 100644 index 0000000000000..9969ebcc44082 --- /dev/null +++ b/registry/v2/routes_test.go @@ -0,0 +1,184 @@ +package v2 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gorilla/mux" +) + +type routeTestCase struct { + RequestURI string + Vars map[string]string + RouteName string + StatusCode int +} + +// TestRouter registers a test handler with all the routes and ensures that +// each route returns the expected path variables. Not method verification is +// present. This not meant to be exhaustive but as check to ensure that the +// expected variables are extracted. +// +// This may go away as the application structure comes together. +func TestRouter(t *testing.T) { + + router := Router() + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testCase := routeTestCase{ + RequestURI: r.RequestURI, + Vars: mux.Vars(r), + RouteName: mux.CurrentRoute(r).GetName(), + } + + enc := json.NewEncoder(w) + + if err := enc.Encode(testCase); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + // Startup test server + server := httptest.NewServer(router) + + for _, testcase := range []routeTestCase{ + { + RouteName: RouteNameBase, + RequestURI: "/v2/", + Vars: map[string]string{}, + }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/tag", + Vars: map[string]string{ + "name": "foo/bar", + "tag": "tag", + }, + }, + { + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/tags/list", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "tarsum.dev+foo:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "sha256:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlobUpload, + RequestURI: "/v2/foo/bar/blobs/uploads/", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/uuid", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "uuid", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + }, + }, + { + // Check ambiguity: ensure we can distinguish between tags for + // "foo/bar/image/image" and image for "foo/bar/image" with tag + // "tags" + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/manifests/tags", + Vars: map[string]string{ + "name": "foo/bar/manifests", + "tag": "tags", + }, + }, + { + // This case presents an ambiguity between foo/bar with tag="tags" + // and list tags for "foo/bar/manifest" + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/manifests/tags/list", + Vars: map[string]string{ + "name": "foo/bar/manifests", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + StatusCode: http.StatusNotFound, + }, + } { + // Register the endpoint + router.GetRoute(testcase.RouteName).Handler(testHandler) + u := server.URL + testcase.RequestURI + + resp, err := http.Get(u) + + if err != nil { + t.Fatalf("error issuing get request: %v", err) + } + + if testcase.StatusCode == 0 { + // Override default, zero-value + testcase.StatusCode = http.StatusOK + } + + if resp.StatusCode != testcase.StatusCode { + t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) + } + + if testcase.StatusCode != http.StatusOK { + // We don't care about json response. + continue + } + + dec := json.NewDecoder(resp.Body) + + var actualRouteInfo routeTestCase + if err := dec.Decode(&actualRouteInfo); err != nil { + t.Fatalf("error reading json response: %v", err) + } + // Needs to be set out of band + actualRouteInfo.StatusCode = resp.StatusCode + + if actualRouteInfo.RouteName != testcase.RouteName { + t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) + } + + if !reflect.DeepEqual(actualRouteInfo, testcase) { + t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) + } + } + +} diff --git a/registry/v2/urls.go b/registry/v2/urls.go new file mode 100644 index 0000000000000..72f44299ab452 --- /dev/null +++ b/registry/v2/urls.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/docker/docker-registry/digest" + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { + u := &url.URL{ + Scheme: r.URL.Scheme, + Host: r.Host, + } + + return NewURLBuilder(u) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and tag. +func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + manifestURL, err := route.URL("name", name, "tag", tag) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", name, "digest", dgst.String()) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name, "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) *mux.Route { + route := new(mux.Route) + *route = *ub.router.GetRoute(name) // clone the route + + return route. + Schemes(ub.root.Scheme). + Host(ub.root.Host) +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} + +// appendValues appends the parameters to the url. Panics if the string is not +// a url. +func appendValues(u string, values ...url.Values) string { + up, err := url.Parse(u) + + if err != nil { + panic(err) // should never happen + } + + return appendValuesURL(up, values...).String() +} diff --git a/registry/v2/urls_test.go b/registry/v2/urls_test.go new file mode 100644 index 0000000000000..a9590dba90a89 --- /dev/null +++ b/registry/v2/urls_test.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "net/url" + "testing" +) + +type urlBuilderTestCase struct { + description string + expected string + build func() (string, error) +} + +// TestURLBuilder tests the various url building functions, ensuring they are +// returning the expected values. +func TestURLBuilder(t *testing.T) { + + root := "http://localhost:5000/" + urlBuilder, err := NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testcase := range []struct { + description string + expected string + build func() (string, error) + }{ + { + description: "test base url", + expected: "http://localhost:5000/v2/", + build: urlBuilder.BuildBaseURL, + }, + { + description: "test tags url", + expected: "http://localhost:5000/v2/foo/bar/tags/list", + build: func() (string, error) { + return urlBuilder.BuildTagsURL("foo/bar") + }, + }, + { + description: "test manifest url", + expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + build: func() (string, error) { + return urlBuilder.BuildManifestURL("foo/bar", "tag") + }, + }, + { + description: "build blob url", + expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + build: func() (string, error) { + return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") + }, + }, + { + description: "build blob upload url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar") + }, + }, + { + description: "build blob upload url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + { + description: "build blob upload chunk url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") + }, + }, + { + description: "build blob upload chunk url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + } { + u, err := testcase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testcase.description, err) + } + + if u != testcase.expected { + t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) + } + } + +} From 9e8b9de554edcdb04c2c0b07c6eba69221506e68 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 15 Dec 2014 12:42:52 -0800 Subject: [PATCH 09/26] Remove dependencies on registry packages Because docker core cannot vendor non-master Go dependencies, we need to remove dependencies on registry package. The definition of digest.Digest has been changed to a string and the regular expressions have been ported from docker-registry/common library. We'll likely change this be dependent on the registry in the future when the API stabilizies and use of the master branch becomes the norm. Signed-off-by: Stephen J Day --- registry/v2/errors_test.go | 4 +--- registry/v2/regexp.go | 19 +++++++++++++++++++ registry/v2/routes.go | 15 ++++++--------- registry/v2/urls.go | 5 ++--- 4 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 registry/v2/regexp.go diff --git a/registry/v2/errors_test.go b/registry/v2/errors_test.go index d2fc091acad54..4a80cdfe2d5d6 100644 --- a/registry/v2/errors_test.go +++ b/registry/v2/errors_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "reflect" "testing" - - "github.com/docker/docker-registry/digest" ) // TestErrorCodes ensures that error code format, mappings and @@ -61,7 +59,7 @@ func TestErrorsManagement(t *testing.T) { errs.Push(ErrorCodeDigestInvalid) errs.Push(ErrorCodeBlobUnknown, - map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + map[string]string{"digest": "sometestblobsumdoesntmatter"}) p, err := json.Marshal(errs) diff --git a/registry/v2/regexp.go b/registry/v2/regexp.go new file mode 100644 index 0000000000000..b7e95b9ff33ae --- /dev/null +++ b/registry/v2/regexp.go @@ -0,0 +1,19 @@ +package v2 + +import "regexp" + +// This file defines regular expressions for use in route definition. These +// are also defined in the registry code base. Until they are in a common, +// shared location, and exported, they must be repeated here. + +// RepositoryNameComponentRegexp restricts registtry path components names to +// start with at least two letters or numbers, with following parts able to +// separated by one period, dash or underscore. +var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) + +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// 5 path components, separated by a forward slash. +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) + +// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. +var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/registry/v2/routes.go b/registry/v2/routes.go index 7ebe61d665d46..08f36e2f712bb 100644 --- a/registry/v2/routes.go +++ b/registry/v2/routes.go @@ -1,9 +1,6 @@ package v2 -import ( - "github.com/docker/docker-registry/common" - "github.com/gorilla/mux" -) +import "github.com/gorilla/mux" // The following are definitions of the name under which all V2 routes are // registered. These symbols can be used to look up a route based on the name. @@ -40,29 +37,29 @@ func Router() *mux.Router { // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). Name(RouteNameTags) // GET /v2//blob/ Layer Fetch the blob identified by digest. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). Name(RouteNameBlob) // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). Name(RouteNameBlobUpload) // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). Name(RouteNameBlobUploadChunk) return router diff --git a/registry/v2/urls.go b/registry/v2/urls.go index 72f44299ab452..19ef06fa12281 100644 --- a/registry/v2/urls.go +++ b/registry/v2/urls.go @@ -4,7 +4,6 @@ import ( "net/http" "net/url" - "github.com/docker/docker-registry/digest" "github.com/gorilla/mux" ) @@ -88,10 +87,10 @@ func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { } // BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { +func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { route := ub.cloneRoute(RouteNameBlob) - layerURL, err := route.URL("name", name, "digest", dgst.String()) + layerURL, err := route.URL("name", name, "digest", dgst) if err != nil { return "", err } From 2f554a7edb67bad0ab0f5558bd61e9b996ff9b4e Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 16 Dec 2014 16:57:37 -0800 Subject: [PATCH 10/26] Update push and pull to registry 2.1 specification Signed-off-by: Derek McGowan --- graph/manifest.go | 50 ++++---- graph/pull.go | 21 ++-- graph/push.go | 65 ++++++----- registry/auth.go | 53 +++++++++ registry/session_v2.go | 256 +++++++++++++---------------------------- utils/jsonmessage.go | 3 + 6 files changed, 208 insertions(+), 240 deletions(-) diff --git a/graph/manifest.go b/graph/manifest.go index ddcb22b650cce..54d6083cba765 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -2,6 +2,7 @@ package graph import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -24,33 +25,47 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { } // Resolve the Repository name from fqn to endpoint + name - _, remoteName, err := registry.ResolveRepositoryName(name) + repoInfo, err := registry.ParseRepositoryInfo(name) if err != nil { return job.Error(err) } + manifestBytes, err := s.newManifest(name, repoInfo.RemoteName, tag) + if err != nil { + return job.Error(err) + } + + _, err = job.Stdout.Write(manifestBytes) + if err != nil { + return job.Error(err) + } + + return engine.StatusOK +} + +func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) { manifest := ®istry.ManifestData{ Name: remoteName, Tag: tag, SchemaVersion: 1, } - localRepo, exists := s.Repositories[name] + localRepo, exists := s.Repositories[localName] if !exists { - return job.Errorf("Repo does not exist: %s", name) + return nil, fmt.Errorf("Repo does not exist: %s", localName) } layerId, exists := localRepo[tag] if !exists { - return job.Errorf("Tag does not exist for %s: %s", name, tag) + return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) } layersSeen := make(map[string]bool) layer, err := s.graph.Get(layerId) if err != nil { - return job.Error(err) + return nil, err } if layer.Config == nil { - return job.Errorf("Missing layer configuration") + return nil, errors.New("Missing layer configuration") } manifest.Architecture = layer.Architecture manifest.FSLayers = make([]*registry.FSLayer, 0, 4) @@ -60,7 +75,7 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { for ; layer != nil; layer, err = layer.GetParent() { if err != nil { - return job.Error(err) + return nil, err } if layersSeen[layer.ID] { @@ -69,21 +84,21 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { if layer.Config != nil && metadata.Image != layer.ID { err = runconfig.Merge(&metadata, layer.Config) if err != nil { - return job.Error(err) + return nil, err } } archive, err := layer.TarLayer() if err != nil { - return job.Error(err) + return nil, err } - tarSum, err := tarsum.NewTarSum(archive, true, tarsum.VersionDev) + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1) if err != nil { - return job.Error(err) + return nil, err } if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { - return job.Error(err) + return nil, err } tarId := tarSum.Sum(nil) @@ -94,20 +109,15 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { layersSeen[layer.ID] = true jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) if err != nil { - return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) + return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) } manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) } manifestBytes, err := json.MarshalIndent(manifest, "", " ") if err != nil { - return job.Error(err) - } - - _, err = job.Stdout.Write(manifestBytes) - if err != nil { - return job.Error(err) + return nil, err } - return engine.StatusOK + return manifestBytes, nil } diff --git a/graph/pull.go b/graph/pull.go index 587eb5f500b90..b138793d1f4c7 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -133,7 +133,12 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Errorf("error updating trust base graph: %s", err) } - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { + auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) + if err != nil { + return job.Errorf("error getting authorization: %s", err) + } + + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } @@ -423,23 +428,23 @@ type downloadInfo struct { err chan error } -func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { +func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) error { var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) - tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, nil) + tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, auth) if err != nil { return err } for _, t := range tags { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true @@ -454,9 +459,9 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out return nil } -func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) { +func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) - manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, nil) + manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, auth) if err != nil { return false, err } @@ -525,7 +530,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } - r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, nil) + r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, auth) if err != nil { return err } diff --git a/graph/push.go b/graph/push.go index 653b0eccb5843..4fe737ff3355b 100644 --- a/graph/push.go +++ b/graph/push.go @@ -229,26 +229,24 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err2) } - var isOfficial bool - if endpoint.String() == registry.IndexServerAddress() { - isOfficial = isOfficialName(remoteName) - if isOfficial && strings.IndexRune(remoteName, '/') == -1 { - remoteName = "library/" + remoteName - } - } - if len(tag) == 0 { tag = DEFAULTTAG } - if isOfficial || endpoint.Version == registry.APIVersion2 { + + if repoInfo.Official || endpoint.Version == registry.APIVersion2 { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { return job.Errorf("error updating trust base graph: %s", err) } - repoData, err := r.PushImageJSONIndex(remoteName, []*registry.ImgData{}, false, nil) + // Get authentication type + auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) if err != nil { - return job.Error(err) + return job.Errorf("error getting authorization: %s", err) + } + + if len(manifestBytes) == 0 { + // TODO Create manifest and sign } // try via manifest @@ -298,13 +296,13 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } // Call mount blob - exists, err := r.PostV2ImageMountBlob(remoteName, sumParts[0], manifestSum, repoData.Tokens) + exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) } if !exists { - _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) @@ -316,35 +314,36 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } // push the manifest - err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader([]byte(manifestBytes)), repoData.Tokens) + err = r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) if err != nil { return job.Error(err) } // done, no fallback to V1 return engine.StatusOK - } + } else { - if err != nil { - reposLen := 1 - if tag == "" { - reposLen = len(s.Repositories[repoInfo.LocalName]) - } - job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) - // If it fails, try to get the repository - if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { - if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { - return job.Error(err) + if err != nil { + reposLen := 1 + if tag == "" { + reposLen = len(s.Repositories[repoInfo.LocalName]) + } + job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) + // If it fails, try to get the repository + if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { + if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK } - return engine.StatusOK + return job.Error(err) } - return job.Error(err) - } - var token []string - job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) - if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { - return job.Error(err) + var token []string + job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) + if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK } - return engine.StatusOK } diff --git a/registry/auth.go b/registry/auth.go index 2044236cfb923..b138fb530d1ef 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -37,6 +37,59 @@ type ConfigFile struct { rootPath string } +type RequestAuthorization struct { + Token string + Username string + Password string +} + +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { + var auth RequestAuthorization + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + factory := HTTPRequestFactory(nil) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + auth.Username = authConfig.Username + auth.Password = authConfig.Password + case "bearer": + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return nil, err + } + + auth.Token = token + default: + log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + return &auth, nil +} + +func (auth *RequestAuthorization) Authorize(req *http.Request) { + if auth.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) + } else if auth.Username != "" && auth.Password != "" { + req.SetBasicAuth(auth.Username, auth.Password) + } +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/registry/session_v2.go b/registry/session_v2.go index 86d0c228a772b..407c5f3a23d62 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -9,100 +9,34 @@ import ( "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" - "github.com/gorilla/mux" ) -func newV2RegistryRouter() *mux.Router { - router := mux.NewRouter() +var registryURLBuilder *v2.URLBuilder - v2Router := router.PathPrefix("/v2/").Subrouter() - - // Version Info - v2Router.Path("/version").Name("version") - - // Image Manifests - v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") - - // List Image Tags - v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") - - // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") - - // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") - - // Mounting a blob in an image - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") - - return router -} - -// APIVersion2 /v2/ -var v2HTTPRoutes = newV2RegistryRouter() - -func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { - route := v2HTTPRoutes.Get(routeName) - if route == nil { - return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) - } - - varReplace := make([]string, 0, len(vars)*2) - for key, val := range vars { - varReplace = append(varReplace, key, val) - } - - routePath, err := route.URLPath(varReplace...) - if err != nil { - return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) - } +func init() { u, err := url.Parse(REGISTRYSERVER) if err != nil { - return nil, fmt.Errorf("invalid registry url: %s", err) + panic(fmt.Errorf("invalid registry url: %s", err)) } - - return &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - Path: routePath.Path, - }, nil + registryURLBuilder = v2.NewURLBuilder(u) } -// V2 Provenance POC - -func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { - routeURL, err := getV2URL(r.indexEndpoint, "version", nil) - if err != nil { - return nil, err - } - - method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) - } - - decoder := json.NewDecoder(res.Body) - versionInfo := new(RegistryInfo) +func getV2Builder(e *Endpoint) *v2.URLBuilder { + return registryURLBuilder +} - err = decoder.Decode(versionInfo) - if err != nil { - return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) +// GetV2Authorization gets the authorization needed to the given image +// If readonly access is requested, then only the authorization may +// only be used for Get operations. +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { + scopes := []string{"pull"} + if !readOnly { + scopes = append(scopes, "push") } - return versionInfo, nil + return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) } // @@ -112,25 +46,20 @@ func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -155,26 +84,20 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) // - Succeeded to mount for this image scope // - Failed with no error (So continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } - method := "POST" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + method := "HEAD" + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return false, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return false, err @@ -191,25 +114,19 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return err @@ -226,25 +143,19 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -267,85 +178,76 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) + if err != nil { + return err } - routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { - return "", err + return err } + auth.Authorize(req) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + location := res.Header.Get("Location") + method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) + log.Debugf("[registry] Calling %q %s", method, location) + req, err = r.reqFactory.NewRequest(method, location, blobRdr) if err != nil { - return "", err + return err } - setTokenAuth(req, token) - req.Header.Set("X-Tarsum", sumStr) - res, _, err := r.doRequest(req) + queryParams := url.Values{} + queryParams.Add("digest", sumType+":"+sumStr) + req.URL.RawQuery = queryParams.Encode() + auth.Authorize(req) + res, _, err = r.doRequest(req) if err != nil { - return "", err + return err } defer res.Body.Close() + if res.StatusCode != 201 { if res.StatusCode == 401 { - return "", errLoginRequired + return errLoginRequired } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) - } - - type sumReturn struct { - Checksum string `json:"checksum"` - } - - decoder := json.NewDecoder(res.Body) - var sumInfo sumReturn - - err = decoder.Decode(&sumInfo) - if err != nil { - return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) } - if sumInfo.Checksum != sumStr { - return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) - } - - // XXX this is a json struct from the registry, with its checksum - return sumInfo.Checksum, nil + return nil } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) if err != nil { return err } method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) if err != nil { return err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return err } + b, _ := ioutil.ReadAll(res.Body) res.Body.Close() - if res.StatusCode != 201 { + if res.StatusCode != 200 { if res.StatusCode == 401 { return errLoginRequired } + log.Debugf("Unexpected response from server: %q %#v", b, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } @@ -353,24 +255,20 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { - vars := map[string]string{ - "imagename": imageName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) +func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index a2bbbcf4d4b6e..74d3112719fb7 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -50,6 +50,9 @@ func (p *JSONProgress) String() string { } total := units.HumanSize(float64(p.Total)) percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if percentage > 50 { + percentage = 50 + } if width > 110 { // this number can't be negetive gh#7136 numSpaces := 0 From 78b6d893b062706eef55bdd186d8dcd1de538b45 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 14:44:18 -0800 Subject: [PATCH 11/26] Allow private V2 registry endpoints Signed-off-by: Derek McGowan --- graph/pull.go | 2 +- graph/push.go | 12 ++++++------ registry/config.go | 2 +- registry/endpoint.go | 2 ++ registry/session_v2.go | 32 +++++++++++++++++++------------- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index b138793d1f4c7..0b75881cdefbe 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -127,7 +127,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } - if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Official || endpoint.Version == registry.APIVersion2) { + if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { return job.Errorf("error updating trust base graph: %s", err) diff --git a/graph/push.go b/graph/push.go index 4fe737ff3355b..f7430b2caf066 100644 --- a/graph/push.go +++ b/graph/push.go @@ -233,13 +233,14 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { tag = DEFAULTTAG } - if repoInfo.Official || endpoint.Version == registry.APIVersion2 { - j := job.Eng.Job("trust_update_base") - if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) + if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { + if repoInfo.Official { + j := job.Eng.Job("trust_update_base") + if err = j.Run(); err != nil { + return job.Errorf("error updating trust base graph: %s", err) + } } - // Get authentication type auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) if err != nil { return job.Errorf("error getting authorization: %s", err) @@ -322,7 +323,6 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { // done, no fallback to V1 return engine.StatusOK } else { - if err != nil { reposLen := 1 if tag == "" { diff --git a/registry/config.go b/registry/config.go index b5652b15d8f6e..4d13aaea3507c 100644 --- a/registry/config.go +++ b/registry/config.go @@ -23,7 +23,7 @@ type Options struct { const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v2/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" diff --git a/registry/endpoint.go b/registry/endpoint.go index 5c5b0520001b2..9a783f1f05544 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -10,6 +10,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" ) // for mocking in unit tests @@ -103,6 +104,7 @@ type Endpoint struct { Version APIVersion IsSecure bool AuthChallenges []*AuthorizationChallenge + URLBuilder *v2.URLBuilder } // Get the formated URL for the root of this registry Endpoint diff --git a/registry/session_v2.go b/registry/session_v2.go index 407c5f3a23d62..2304a61344152 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -13,30 +13,36 @@ import ( "github.com/docker/docker/utils" ) -var registryURLBuilder *v2.URLBuilder - -func init() { - u, err := url.Parse(REGISTRYSERVER) - if err != nil { - panic(fmt.Errorf("invalid registry url: %s", err)) - } - registryURLBuilder = v2.NewURLBuilder(u) -} - func getV2Builder(e *Endpoint) *v2.URLBuilder { - return registryURLBuilder + if e.URLBuilder == nil { + e.URLBuilder = v2.NewURLBuilder(e.URL) + } + return e.URLBuilder } // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) + var registry *Endpoint + if r.indexEndpoint.URL.Host == IndexServerURL.Host { + registry, err = NewEndpoint(REGISTRYSERVER, nil) + if err != nil { + return + } + } else { + registry = r.indexEndpoint + } + registry.URLBuilder = v2.NewURLBuilder(registry.URL) + r.indexEndpoint = registry + + log.Debugf("Getting authorization for %s %s", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) } // From e4823e89761ad5b034f7001aea16d4dec520553b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 16:14:04 -0800 Subject: [PATCH 12/26] Get token on each request Signed-off-by: Derek McGowan --- registry/auth.go | 60 ++++++++++++++++++++++++++---------------- registry/session_v2.go | 34 +++++++++++++++++------- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/registry/auth.go b/registry/auth.go index b138fb530d1ef..1e1c7ddb82c5e 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -38,56 +38,70 @@ type ConfigFile struct { } type RequestAuthorization struct { - Token string - Username string - Password string + authConfig *AuthConfig + registryEndpoint *Endpoint + resource string + scope string + actions []string } -func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { - var auth RequestAuthorization +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { + return &RequestAuthorization{ + authConfig: authConfig, + registryEndpoint: registryEndpoint, + resource: resource, + scope: scope, + actions: actions, + } +} +func (auth *RequestAuthorization) getToken() (string, error) { + // TODO check if already has token and before expiration client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - }, + Proxy: http.ProxyFromEnvironment}, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } factory := HTTPRequestFactory(nil) - for _, challenge := range registryEndpoint.AuthChallenges { - log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) - + for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { case "basic": - auth.Username = authConfig.Username - auth.Password = authConfig.Password + // no token necessary case "bearer": + log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) params := map[string]string{} for k, v := range challenge.Parameters { params[k] = v } - params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) if err != nil { - return nil, err + return "", err } + // TODO cache token and set expiration to one minute from now - auth.Token = token + return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - - return &auth, nil + // TODO no expiration, do not reattempt to get a token + return "", nil } -func (auth *RequestAuthorization) Authorize(req *http.Request) { - if auth.Token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) - } else if auth.Username != "" && auth.Password != "" { - req.SetBasicAuth(auth.Username, auth.Password) +func (auth *RequestAuthorization) Authorize(req *http.Request) error { + token, err := auth.getToken() + if err != nil { + return err + } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { + req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) } + return nil } // create a base64 encoded auth string to store in config diff --git a/registry/session_v2.go b/registry/session_v2.go index 2304a61344152..491cd2c6e040a 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -42,7 +42,7 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req r.indexEndpoint = registry log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil } // @@ -65,7 +65,9 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -103,7 +105,9 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return false, err @@ -132,7 +136,9 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -161,7 +167,9 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -196,7 +204,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -212,7 +222,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err = r.doRequest(req) if err != nil { return err @@ -242,7 +254,9 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -274,7 +288,9 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err From 450b5eb389b9bcb115bfc292a357141c2e563151 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 22 Dec 2014 14:58:08 -0800 Subject: [PATCH 13/26] Correctly check and propagate errors in v2 session Signed-off-by: Stephen J Day --- registry/session_v2.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/registry/session_v2.go b/registry/session_v2.go index 491cd2c6e040a..411df46e35c19 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -65,7 +65,7 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) @@ -105,8 +105,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return false, err } res, _, err := r.doRequest(req) if err != nil { @@ -136,8 +136,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -167,8 +167,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return nil, 0, err } res, _, err := r.doRequest(req) if err != nil { @@ -204,8 +204,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -222,8 +222,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err = r.doRequest(req) if err != nil { @@ -254,8 +254,8 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -288,7 +288,7 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) From 00dd323f5bc67d34aeafb910eb02091c6e093bd7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 22 Dec 2014 18:58:01 -0800 Subject: [PATCH 14/26] Fix tests Signed-off-by: Derek McGowan --- utils/jsonmessage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/jsonmessage_test.go b/utils/jsonmessage_test.go index 0ce9492c9818a..b9103da1a4bdb 100644 --- a/utils/jsonmessage_test.go +++ b/utils/jsonmessage_test.go @@ -30,7 +30,7 @@ func TestProgress(t *testing.T) { } // this number can't be negetive gh#7136 - expected = "[==============================================================>] 50 B/40 B" + expected = "[==================================================>] 50 B/40 B" jp4 := JSONProgress{Current: 50, Total: 40} if jp4.String() != expected { t.Fatalf("Expected %q, got %q", expected, jp4.String()) From ef9cf88527523ab7b4e33115d4ddedef6370ceb6 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 23 Dec 2014 13:40:06 -0800 Subject: [PATCH 15/26] Add Tarsum Calculation during v2 Pull operation While the v2 pull operation is writing the body of the layer blob to disk it now computes the tarsum checksum of the archive before extracting it to the backend storage driver. If the checksum does not match that from the image manifest an error is raised. Also adds more debug logging to the pull operation and fixes existing test cases which were failing. Adds a reverse lookup constructor to the tarsum package so that you can get a tarsum object using a checksum label. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- graph/pull.go | 30 ++++++++++++++++++++++++++++-- image/image.go | 33 +++++++++++++++++---------------- pkg/tarsum/tarsum.go | 39 +++++++++++++++++++++++++++++++++++++++ pkg/tarsum/versioning.go | 17 ++++++++++++----- registry/endpoint.go | 17 ++++++++++++----- registry/session_v2.go | 8 ++++++-- 6 files changed, 114 insertions(+), 30 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index 0b75881cdefbe..88e939a4818f3 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -15,6 +15,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" "github.com/docker/libtrust" @@ -112,6 +113,8 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) + + log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { return job.Error(err) @@ -127,6 +130,10 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } + // Calling the v2 code path might change the session + // endpoint value, so save the original one! + originalSession := *r + if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { @@ -138,6 +145,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Errorf("error getting authorization: %s", err) } + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) @@ -146,8 +154,13 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } else if err != registry.ErrDoesNotExist { log.Errorf("Error from V2 registry: %s", err) } + + log.Debug("image does not exist on v2 registry, falling back to v1") } + r = &originalSession + + log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { return job.Error(err) } @@ -174,7 +187,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo * log.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) if err != nil { - log.Errorf("%v", err) + log.Errorf("unable to get remote tags: %s", err) return err } @@ -535,7 +548,20 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } defer r.Close() - io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + // Wrap the reader with the appropriate TarSum reader. + tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType) + if err != nil { + return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err) + } + + io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil)) + + if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) { + return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum) + } out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) diff --git a/image/image.go b/image/image.go index 8cd9aa3755130..7664602cd8d75 100644 --- a/image/image.go +++ b/image/image.go @@ -94,28 +94,29 @@ func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error // If layerData is not nil, unpack it into the new layer if layerData != nil { - layerDataDecompressed, err := archive.DecompressStream(layerData) - if err != nil { - return err - } + // If the image doesn't have a checksum, we should add it. The layer + // checksums are verified when they are pulled from a remote, but when + // a container is committed it should be added here. + if img.Checksum == "" { + layerDataDecompressed, err := archive.DecompressStream(layerData) + if err != nil { + return err + } + defer layerDataDecompressed.Close() - defer layerDataDecompressed.Close() + if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { + return err + } - if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { - return err - } + if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { + return err + } - if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { + img.Checksum = layerTarSum.Sum(nil) + } else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { return err } - checksum := layerTarSum.Sum(nil) - - if img.Checksum != "" && img.Checksum != checksum { - log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum) - } - - img.Checksum = checksum } img.Size = size diff --git a/pkg/tarsum/tarsum.go b/pkg/tarsum/tarsum.go index c9f1315cf524b..c6a7294e74e85 100644 --- a/pkg/tarsum/tarsum.go +++ b/pkg/tarsum/tarsum.go @@ -3,8 +3,11 @@ package tarsum import ( "bytes" "compress/gzip" + "crypto" "crypto/sha256" "encoding/hex" + "errors" + "fmt" "hash" "io" "strings" @@ -39,6 +42,30 @@ func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) return ts, err } +// Create a new TarSum using the provided TarSum version+hash label. +func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) { + parts := strings.SplitN(label, "+", 2) + if len(parts) != 2 { + return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}") + } + + versionName, hashName := parts[0], parts[1] + + version, ok := tarSumVersionsByName[versionName] + if !ok { + return nil, fmt.Errorf("unknown TarSum version name: %q", versionName) + } + + hashConfig, ok := standardHashConfigs[hashName] + if !ok { + return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName) + } + + tHash := NewTHash(hashConfig.name, hashConfig.hash.New) + + return NewTarSumHash(r, disableCompression, version, tHash) +} + // TarSum is the generic interface for calculating fixed time // checksums of a tar archive type TarSum interface { @@ -89,6 +116,18 @@ func NewTHash(name string, h func() hash.Hash) THash { return simpleTHash{n: name, h: h} } +type tHashConfig struct { + name string + hash crypto.Hash +} + +var ( + standardHashConfigs = map[string]tHashConfig{ + "sha256": {name: "sha256", hash: crypto.SHA256}, + "sha512": {name: "sha512", hash: crypto.SHA512}, + } +) + // TarSum default is "sha256" var DefaultTHash = NewTHash("sha256", sha256.New) diff --git a/pkg/tarsum/versioning.go b/pkg/tarsum/versioning.go index 3a656612ff63a..be1d07040f9ee 100644 --- a/pkg/tarsum/versioning.go +++ b/pkg/tarsum/versioning.go @@ -31,11 +31,18 @@ func GetVersions() []Version { return v } -var tarSumVersions = map[Version]string{ - Version0: "tarsum", - Version1: "tarsum.v1", - VersionDev: "tarsum.dev", -} +var ( + tarSumVersions = map[Version]string{ + Version0: "tarsum", + Version1: "tarsum.v1", + VersionDev: "tarsum.dev", + } + tarSumVersionsByName = map[string]Version{ + "tarsum": Version0, + "tarsum.v1": Version1, + "tarsum.dev": VersionDev, + } +) func (tsv Version) String() string { return tarSumVersions[tsv] diff --git a/registry/endpoint.go b/registry/endpoint.go index 9a783f1f05544..9ca9ed8b9a6e2 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -47,16 +47,23 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { if err != nil { return nil, err } + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + + return endpoint, nil +} +func validateEndpoint(endpoint *Endpoint) error { log.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - if index.Secure { + if endpoint.IsSecure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. @@ -65,13 +72,13 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { var err2 error if _, err2 = endpoint.Ping(); err2 == nil { - return endpoint, nil + return nil } - return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } - return endpoint, nil + return nil } func newEndpoint(address string, secure bool) (*Endpoint, error) { diff --git a/registry/session_v2.go b/registry/session_v2.go index 411df46e35c19..031122dcf6d1f 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -30,8 +30,12 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req } var registry *Endpoint - if r.indexEndpoint.URL.Host == IndexServerURL.Host { - registry, err = NewEndpoint(REGISTRYSERVER, nil) + if r.indexEndpoint.String() == IndexServerAddress() { + registry, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(registry) if err != nil { return } From 2e4bb3b76d36520d1816911152e27e077782636f Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 2 Jan 2015 11:13:11 -0800 Subject: [PATCH 16/26] Refactor from feedback Signed-off-by: Derek McGowan (github: dmcgowan) --- docker/docker.go | 16 +-------- graph/manifest.go | 79 ++++++++++++++++++++++++++++++++++++++++-- graph/pull.go | 67 ++--------------------------------- graph/push.go | 10 ++---- registry/session_v2.go | 3 +- 5 files changed, 82 insertions(+), 93 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 84ffeace9a990..92f5f1460327e 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "strings" log "github.com/Sirupsen/logrus" @@ -16,7 +15,6 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/utils" - "github.com/docker/libtrust" ) const ( @@ -79,22 +77,10 @@ func main() { } protoAddrParts := strings.SplitN(flHosts[0], "://", 2) - err := os.MkdirAll(path.Dir(*flTrustKey), 0700) + trustKey, err := api.LoadOrCreateTrustKey(*flTrustKey) if err != nil { log.Fatal(err) } - trustKey, err := libtrust.LoadKeyFile(*flTrustKey) - if err == libtrust.ErrKeyFileDoesNotExist { - trustKey, err = libtrust.GenerateECP256PrivateKey() - if err != nil { - log.Fatalf("Error generating key: %s", err) - } - if err := libtrust.SaveKey(*flTrustKey, trustKey); err != nil { - log.Fatalf("Error saving key file: %s", err) - } - } else if err != nil { - log.Fatalf("Error loading key file: %s", err) - } var ( cli *client.DockerCli diff --git a/graph/manifest.go b/graph/manifest.go index 54d6083cba765..3d4ab1c5dee78 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -1,6 +1,7 @@ package graph import ( + "bytes" "encoding/json" "errors" "fmt" @@ -8,10 +9,12 @@ import ( "io/ioutil" "path" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + "github.com/docker/libtrust" ) func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { @@ -49,11 +52,15 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error Tag: tag, SchemaVersion: 1, } - localRepo, exists := s.Repositories[localName] - if !exists { + localRepo, err := s.Get(localName) + if err != nil { + return nil, err + } + if localRepo == nil { return nil, fmt.Errorf("Repo does not exist: %s", localName) } + // Get the top-most layer id which the tag points to layerId, exists := localRepo[tag] if !exists { return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) @@ -102,7 +109,6 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } tarId := tarSum.Sum(nil) - // Save tarsum to image json manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) @@ -121,3 +127,70 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error return manifestBytes, nil } + +func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { + sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") + if err != nil { + return nil, false, fmt.Errorf("error parsing payload: %s", err) + } + + keys, err := sig.Verify() + if err != nil { + return nil, false, fmt.Errorf("error verifying payload: %s", err) + } + + payload, err := sig.Payload() + if err != nil { + return nil, false, fmt.Errorf("error retrieving payload: %s", err) + } + + var manifest registry.ManifestData + if err := json.Unmarshal(payload, &manifest); err != nil { + return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) + } + if manifest.SchemaVersion != 1 { + return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) + } + + var verified bool + for _, key := range keys { + job := eng.Job("trust_key_check") + b, err := key.MarshalJSON() + if err != nil { + return nil, false, fmt.Errorf("error marshalling public key: %s", err) + } + namespace := manifest.Name + if namespace[0] != '/' { + namespace = "/" + namespace + } + stdoutBuffer := bytes.NewBuffer(nil) + + job.Args = append(job.Args, namespace) + job.Setenv("PublicKey", string(b)) + // Check key has read/write permission (0x03) + job.SetenvInt("Permission", 0x03) + job.Stdout.Add(stdoutBuffer) + if err = job.Run(); err != nil { + return nil, false, fmt.Errorf("error running key check: %s", err) + } + result := engine.Tail(stdoutBuffer, 1) + log.Debugf("Key check result: %q", result) + if result == "verified" { + verified = true + } + } + + return &manifest, verified, nil +} + +func checkValidManifest(manifest *registry.ManifestData) error { + if len(manifest.FSLayers) != len(manifest.History) { + return fmt.Errorf("length of history not equal to number of layers") + } + + if len(manifest.FSLayers) == 0 { + return fmt.Errorf("no FSLayers in manifest") + } + + return nil +} diff --git a/graph/pull.go b/graph/pull.go index 88e939a4818f3..b2710e9b6859e 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -1,8 +1,6 @@ package graph import ( - "bytes" - "encoding/json" "fmt" "io" "io/ioutil" @@ -18,63 +16,8 @@ import ( "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" - "github.com/docker/libtrust" ) -func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { - sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") - if err != nil { - return nil, false, fmt.Errorf("error parsing payload: %s", err) - } - keys, err := sig.Verify() - if err != nil { - return nil, false, fmt.Errorf("error verifying payload: %s", err) - } - - payload, err := sig.Payload() - if err != nil { - return nil, false, fmt.Errorf("error retrieving payload: %s", err) - } - - var manifest registry.ManifestData - if err := json.Unmarshal(payload, &manifest); err != nil { - return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) - } - if manifest.SchemaVersion != 1 { - return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) - } - - var verified bool - for _, key := range keys { - job := eng.Job("trust_key_check") - b, err := key.MarshalJSON() - if err != nil { - return nil, false, fmt.Errorf("error marshalling public key: %s", err) - } - namespace := manifest.Name - if namespace[0] != '/' { - namespace = "/" + namespace - } - stdoutBuffer := bytes.NewBuffer(nil) - - job.Args = append(job.Args, namespace) - job.Setenv("PublicKey", string(b)) - // Check key has read/write permission (0x03) - job.SetenvInt("Permission", 0x03) - job.Stdout.Add(stdoutBuffer) - if err = job.Run(); err != nil { - return nil, false, fmt.Errorf("error running key check: %s", err) - } - result := engine.Tail(stdoutBuffer, 1) - log.Debugf("Key check result: %q", result) - if result == "verified" { - verified = true - } - } - - return &manifest, verified, nil -} - func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 && n != 2 { return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) @@ -113,7 +56,6 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) - log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { @@ -484,8 +426,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return false, fmt.Errorf("error verifying manifest: %s", err) } - if len(manifest.FSLayers) != len(manifest.History) { - return false, fmt.Errorf("length of history not equal to number of layers") + if err := checkValidManifest(manifest); err != nil { + return false, err } if verified { @@ -493,11 +435,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } else { out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) } - - if len(manifest.FSLayers) == 0 { - return false, fmt.Errorf("no blobSums in manifest") - } - downloads := make([]downloadInfo, len(manifest.FSLayers)) for i := len(manifest.FSLayers) - 1; i >= 0; i-- { diff --git a/graph/push.go b/graph/push.go index f7430b2caf066..4c6b79817e836 100644 --- a/graph/push.go +++ b/graph/push.go @@ -250,14 +250,13 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { // TODO Create manifest and sign } - // try via manifest manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) if err != nil { return job.Errorf("error verifying manifest: %s", err) } - if len(manifest.FSLayers) != len(manifest.History) { - return job.Errorf("length of history not equal to number of layers") + if err := checkValidManifest(manifest); err != nil { + return job.Errorf("invalid manifest: %s", err) } if !verified { @@ -276,11 +275,6 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } manifestSum := sumParts[1] - // for each layer, check if it exists ... - // XXX wait this requires having the TarSum of the layer.tar first - // skip this step for now. Just push the layer every time for this naive implementation - //shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string) - img, err := image.NewImgJSON(imgJSON) if err != nil { return job.Errorf("Failed to parse json: %s", err) diff --git a/registry/session_v2.go b/registry/session_v2.go index 031122dcf6d1f..0e03f4a9ceb8a 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "strconv" log "github.com/Sirupsen/logrus" @@ -223,7 +222,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R if err != nil { return err } - queryParams := url.Values{} + queryParams := req.URL.Query() queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() if err := auth.Authorize(req); err != nil { From a75c70111971772f89715cb019e55aa2836957a7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 14:59:12 -0800 Subject: [PATCH 17/26] Update push to sign with the daemon's key when no manifest is given Signed-off-by: Derek McGowan (github: dmcgowan) --- daemon/daemon.go | 12 ++++++------ graph/push.go | 22 +++++++++++++++++++++- graph/tags.go | 5 ++++- graph/tags_unit_test.go | 2 +- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 68f688e6b911a..30ffa2af1caf4 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -897,8 +897,13 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } + trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) + if err != nil { + return nil, err + } + log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, trustKey) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } @@ -961,11 +966,6 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } - trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) - if err != nil { - return nil, err - } - daemon := &Daemon{ ID: trustKey.PublicKey().KeyID(), repository: daemonRepo, diff --git a/graph/push.go b/graph/push.go index 4c6b79817e836..8aae658982084 100644 --- a/graph/push.go +++ b/graph/push.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) // Retrieve the all the images to be uploaded in the correct order @@ -247,7 +248,26 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } if len(manifestBytes) == 0 { - // TODO Create manifest and sign + mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) + if err != nil { + return job.Error(err) + } + js, err := libtrust.NewJSONSignature(mBytes) + if err != nil { + return job.Error(err) + } + + if err = js.Sign(s.trustKey); err != nil { + return job.Error(err) + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return job.Error(err) + } + log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) + + manifestBytes = string(signedBody) } manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) diff --git a/graph/tags.go b/graph/tags.go index 998b447e6c34b..6bdb296cd1814 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const DEFAULTTAG = "latest" @@ -27,6 +28,7 @@ type TagStore struct { path string graph *Graph Repositories map[string]Repository + trustKey libtrust.PrivateKey sync.Mutex // FIXME: move push/pull-related fields // to a helper type @@ -54,7 +56,7 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, key libtrust.PrivateKey) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -63,6 +65,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { store := &TagStore{ path: abspath, graph: graph, + trustKey: key, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 45dad62951745..58ad8ed878345 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -57,7 +57,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil) if err != nil { t.Fatal(err) } From 4667641ab5bd51163e6ea57bdf6085588523f46c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 15:55:29 -0800 Subject: [PATCH 18/26] Fix list tags Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/session_v2.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/registry/session_v2.go b/registry/session_v2.go index 0e03f4a9ceb8a..b08f4cf0d8e16 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -277,6 +277,11 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R return nil } +type remoteTags struct { + name string + tags []string +} + // Given a repository name, returns a json array of string tags func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) @@ -309,10 +314,10 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) } decoder := json.NewDecoder(res.Body) - var tags []string - err = decoder.Decode(&tags) + var remote remoteTags + err = decoder.Decode(&remote) if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return tags, nil + return remote.tags, nil } From 2d8971af964f8a02749478f2ad1972f62678ef55 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 12 Jan 2015 14:17:50 -0800 Subject: [PATCH 19/26] Fix integration test failures Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index b2710e9b6859e..1c4bb9d88ca36 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -79,22 +79,23 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) + log.Errorf("error updating trust base graph: %s", err) } auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) if err != nil { - return job.Errorf("error getting authorization: %s", err) - } + log.Errorf("error getting authorization: %s", err) + } else { - log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { - if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { - log.Errorf("Error logging event 'pull' for %s: %s", logName, err) + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { + if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { + log.Errorf("Error logging event 'pull' for %s: %s", logName, err) + } + return engine.StatusOK + } else if err != registry.ErrDoesNotExist { + log.Errorf("Error from V2 registry: %s", err) } - return engine.StatusOK - } else if err != registry.ErrDoesNotExist { - log.Errorf("Error from V2 registry: %s", err) } log.Debug("image does not exist on v2 registry, falling back to v1") From d9f5757f4f0c2d0bc8deae450abd9d61ba1caf84 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 11:47:40 -0800 Subject: [PATCH 20/26] Install registry V2 in image Signed-off-by: Alexander Morozov --- Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index 86130c4caabb9..a97cd0f47ef7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -130,6 +130,17 @@ RUN set -x \ && git clone -b v1.2 https://github.com/russross/blackfriday.git /go/src/github.com/russross/blackfriday \ && go install -v github.com/cpuguy83/go-md2man +# Install registry +COPY pkg/tarsum /go/src/github.com/docker/docker/pkg/tarsum +# REGISTRY_COMMIT gives us the repeatability guarantees we need +# (so that we're all testing the same version of the registry) +ENV REGISTRY_COMMIT 21a69f53b5c7986b831f33849d551cd59ec8cbd1 +RUN set -x \ + && git clone https://github.com/docker/distribution.git /go/src/github.com/docker/distribution \ + && (cd /go/src/github.com/docker/distribution && git checkout -q $REGISTRY_COMMIT) \ + && go get -d github.com/docker/distribution/cmd/registry \ + && go build -o /go/bin/registry-v2 github.com/docker/distribution/cmd/registry + # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] From 01540689c32814a6724173e3519d4340b178e6db Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 13:26:49 -0800 Subject: [PATCH 21/26] RegistryV2 datastructure for tests Signed-off-by: Alexander Morozov --- integration-cli/registry.go | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 integration-cli/registry.go diff --git a/integration-cli/registry.go b/integration-cli/registry.go new file mode 100644 index 0000000000000..f0ef05cca1afc --- /dev/null +++ b/integration-cli/registry.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" +) + +const v2binary = "registry-v2" + +type testRegistryV2 struct { + URL string + cmd *exec.Cmd + dir string +} + +func newTestRegistryV2(t *testing.T) (*testRegistryV2, error) { + template := `version: 0.1 +loglevel: debug +storage: + filesystem: + rootdirectory: %s +http: + addr: :%s` + tmp, err := ioutil.TempDir("", "registry-test-") + if err != nil { + return nil, err + } + confPath := filepath.Join(tmp, "config.yaml") + config, err := os.Create(confPath) + if err != nil { + return nil, err + } + if _, err := fmt.Fprintf(config, template, tmp, "5000"); err != nil { + os.RemoveAll(tmp) + return nil, err + } + + cmd := exec.Command(v2binary, confPath) + if err := cmd.Start(); err != nil { + os.RemoveAll(tmp) + if os.IsNotExist(err) { + t.Skip() + } + return nil, err + } + return &testRegistryV2{ + cmd: cmd, + dir: tmp, + URL: "localhost:5000", + }, nil +} + +func (r *testRegistryV2) Close() { + r.cmd.Process.Kill() + os.RemoveAll(r.dir) +} From 018edd3a3cbeb3d030c080c9bb320573bec09590 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 14:30:19 -0800 Subject: [PATCH 22/26] Tests for push to registry v2 Signed-off-by: Alexander Morozov --- integration-cli/docker_cli_push_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 0dfd85a9d4538..a8c2ccdbc3891 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -10,29 +10,27 @@ import ( // pulling an image from the central registry should work func TestPushBusyboxImage(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() + reg, err := newTestRegistryV2(t) + if err != nil { + t.Fatal(err) + } + defer reg.Close() + repoName := fmt.Sprintf("%v/dockercli/busybox", reg.URL) // tag the image to upload it tot he private registry - repoName := fmt.Sprintf("%v/busybox", privateRegistryURL) tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) } - + defer deleteImages(repoName) pushCmd := exec.Command(dockerBinary, "push", repoName) if out, _, err := runCommandWithOutput(pushCmd); err != nil { t.Fatalf("pushing the image to the private registry has failed: %s, %v", out, err) } - - deleteImages(repoName) - logDone("push - push busybox to private registry") } // pushing an image without a prefix should throw an error func TestPushUnprefixedRepo(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() pushCmd := exec.Command(dockerBinary, "push", "busybox") if out, _, err := runCommandWithOutput(pushCmd); err == nil { t.Fatalf("pushing an unprefixed repo didn't result in a non-zero exit status: %s", out) From f652e54988fc20fd2bec91646248f851d27d5881 Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Tue, 13 Jan 2015 10:46:32 -0800 Subject: [PATCH 23/26] Add some push test coverage Signed-off-by: Arnaud Porterie --- integration-cli/docker_cli_push_test.go | 63 +++++++++++++++++++++---- integration-cli/docker_utils.go | 8 ++++ integration-cli/registry.go | 6 +-- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index a8c2ccdbc3891..484e5db70b090 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -3,30 +3,28 @@ package main import ( "fmt" "os/exec" + "strings" "testing" + "time" ) -// these tests need a freshly started empty private docker registry - // pulling an image from the central registry should work func TestPushBusyboxImage(t *testing.T) { - reg, err := newTestRegistryV2(t) - if err != nil { - t.Fatal(err) - } - defer reg.Close() - repoName := fmt.Sprintf("%v/dockercli/busybox", reg.URL) + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) // tag the image to upload it tot he private registry tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) } defer deleteImages(repoName) + pushCmd := exec.Command(dockerBinary, "push", repoName) if out, _, err := runCommandWithOutput(pushCmd); err != nil { t.Fatalf("pushing the image to the private registry has failed: %s, %v", out, err) } - logDone("push - push busybox to private registry") + logDone("push - busybox to private registry") } // pushing an image without a prefix should throw an error @@ -35,5 +33,50 @@ func TestPushUnprefixedRepo(t *testing.T) { if out, _, err := runCommandWithOutput(pushCmd); err == nil { t.Fatalf("pushing an unprefixed repo didn't result in a non-zero exit status: %s", out) } - logDone("push - push unprefixed busybox repo --> must fail") + logDone("push - unprefixed busybox repo must fail") +} + +func TestPushUntagged(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + + expected := "does not exist" + pushCmd := exec.Command(dockerBinary, "push", repoName) + if out, _, err := runCommandWithOutput(pushCmd); err == nil { + t.Fatalf("pushing the image to the private registry should have failed: outuput %q", out) + } else if !strings.Contains(out, expected) { + t.Fatalf("pushing the image failed with an unexpected message: expected %q, got %q", expected, out) + } + logDone("push - untagged image") +} + +func TestPushInterrupt(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it tot he private registry + tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) + if out, _, err := runCommandWithOutput(tagCmd); err != nil { + t.Fatalf("image tagging failed: %s, %v", out, err) + } + defer deleteImages(repoName) + + pushCmd := exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + // Interrupt push (yes, we have no idea at what point it will get killed). + time.Sleep(200 * time.Millisecond) + if err := pushCmd.Process.Kill(); err != nil { + t.Fatalf("Failed to kill push process: %v", err) + } + // Try agin + pushCmd = exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + logDone("push - interrupted") } diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index 03d34b6d70ccf..8c03f1f70a487 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -843,3 +843,11 @@ func readContainerFile(containerId, filename string) ([]byte, error) { return content, nil } + +func setupRegistry(t *testing.T) func() { + reg, err := newTestRegistryV2(t) + if err != nil { + t.Fatal(err) + } + return func() { reg.Close() } +} diff --git a/integration-cli/registry.go b/integration-cli/registry.go index f0ef05cca1afc..00ba3030a9dc4 100644 --- a/integration-cli/registry.go +++ b/integration-cli/registry.go @@ -12,7 +12,6 @@ import ( const v2binary = "registry-v2" type testRegistryV2 struct { - URL string cmd *exec.Cmd dir string } @@ -24,7 +23,7 @@ storage: filesystem: rootdirectory: %s http: - addr: :%s` + addr: %s` tmp, err := ioutil.TempDir("", "registry-test-") if err != nil { return nil, err @@ -34,7 +33,7 @@ http: if err != nil { return nil, err } - if _, err := fmt.Fprintf(config, template, tmp, "5000"); err != nil { + if _, err := fmt.Fprintf(config, template, tmp, privateRegistryURL); err != nil { os.RemoveAll(tmp) return nil, err } @@ -50,7 +49,6 @@ http: return &testRegistryV2{ cmd: cmd, dir: tmp, - URL: "localhost:5000", }, nil } From 93b41d668cd21352a4cceea37e8ee7c59aec9ce2 Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Tue, 13 Jan 2015 15:19:44 -0800 Subject: [PATCH 24/26] Test pulling image with aliases Signed-off-by: Arnaud Porterie --- integration-cli/docker_cli_pull_test.go | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index bed015be0ef22..e76f4ee9508b3 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,12 +1,56 @@ package main import ( + "fmt" "os/exec" "strings" "testing" ) -// FIXME: we need a test for pulling all aliases for an image (issue #8141) +// See issue docker/docker#8141 +func TestPullImageWithAliases(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + defer deleteImages(repoName) + + repos := []string{} + for _, tag := range []string{"recent", "fresh"} { + repos = append(repos, fmt.Sprintf("%v:%v", repoName, tag)) + } + + // Tag and push the same image multiple times. + for _, repo := range repos { + if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil { + t.Fatalf("Failed to tag image %v: error %v, output %q", repos, err, out) + } + if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil { + t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out)) + } + } + + // Clear local images store. + args := append([]string{"rmi"}, repos...) + if out, err := exec.Command(dockerBinary, args...).CombinedOutput(); err != nil { + t.Fatalf("Failed to clean images: error %v, output %q", err, string(out)) + } + + // Pull a single tag and verify it doesn't bring down all aliases. + pullCmd := exec.Command(dockerBinary, "pull", repos[0]) + if out, _, err := runCommandWithOutput(pullCmd); err != nil { + t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out) + } + if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil { + t.Fatalf("Image %v was not pulled down", repos[0]) + } + for _, repo := range repos[1:] { + if err := exec.Command(dockerBinary, "inspect", repo).Run(); err == nil { + t.Fatalf("Image %v shouldn't have been pulled down", repo) + } + } + + logDone("pull - image with aliases") +} // pulling an image from the central registry should work func TestPullImageFromCentralRegistry(t *testing.T) { From d4df9d99bd73121ac205ddf91df4bfc66f6f9115 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 13 Jan 2015 15:48:49 -0800 Subject: [PATCH 25/26] Refactor push and pull to move code out of cmd function Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 26 +++--- graph/push.go | 237 ++++++++++++++++++++++++++------------------------ 2 files changed, 133 insertions(+), 130 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index 1c4bb9d88ca36..c70b220cc9880 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -82,20 +82,14 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { log.Errorf("error updating trust base graph: %s", err) } - auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) - if err != nil { - log.Errorf("error getting authorization: %s", err) - } else { - - log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { - if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { - log.Errorf("Error logging event 'pull' for %s: %s", logName, err) - } - return engine.StatusOK - } else if err != registry.ErrDoesNotExist { - log.Errorf("Error from V2 registry: %s", err) + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { + if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { + log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } + return engine.StatusOK + } else if err != registry.ErrDoesNotExist { + log.Errorf("Error from V2 registry: %s", err) } log.Debug("image does not exist on v2 registry, falling back to v1") @@ -384,7 +378,11 @@ type downloadInfo struct { err chan error } -func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) error { +func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { + auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) diff --git a/graph/push.go b/graph/push.go index 8aae658982084..32f1b5359867c 100644 --- a/graph/push.go +++ b/graph/push.go @@ -191,6 +191,105 @@ func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep strin return imgData.Checksum, nil } +func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out io.Writer, repoInfo *registry.RepositoryInfo, manifestBytes, tag string, sf *utils.StreamFormatter) error { + if repoInfo.Official { + j := eng.Job("trust_update_base") + if err := j.Run(); err != nil { + log.Errorf("error updating trust base graph: %s", err) + } + } + + auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } + + // if no manifest is given, generate and sign with the key associated with the local tag store + if len(manifestBytes) == 0 { + mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) + if err != nil { + return err + } + js, err := libtrust.NewJSONSignature(mBytes) + if err != nil { + return err + } + + if err = js.Sign(s.trustKey); err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) + + manifestBytes = string(signedBody) + } + + manifest, verified, err := s.verifyManifest(eng, []byte(manifestBytes)) + if err != nil { + return fmt.Errorf("error verifying manifest: %s", err) + } + + if err := checkValidManifest(manifest); err != nil { + return fmt.Errorf("invalid manifest: %s", err) + } + + if !verified { + log.Debugf("Pushing unverified image") + } + + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { + var ( + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) + ) + + sumParts := strings.SplitN(sumStr, ":", 2) + if len(sumParts) < 2 { + return fmt.Errorf("Invalid checksum: %s", sumStr) + } + manifestSum := sumParts[1] + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return fmt.Errorf("Failed to parse json: %s", err) + } + + img, err = s.graph.Get(img.ID) + if err != nil { + return err + } + + arch, err := img.TarLayer() + if err != nil { + return fmt.Errorf("Could not get tar layer: %s", err) + } + + // Call mount blob + exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + if !exists { + err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), out, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } else { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) + } + } + + // push the manifest + return r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) +} + // FIXME: Allow to interrupt current push when new push of same image is done. func (s *TagStore) CmdPush(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 { @@ -235,129 +334,35 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { - if repoInfo.Official { - j := job.Eng.Job("trust_update_base") - if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) - } - } - - auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) - if err != nil { - return job.Errorf("error getting authorization: %s", err) + err := s.pushV2Repository(r, job.Eng, job.Stdout, repoInfo, manifestBytes, tag, sf) + if err == nil { + return engine.StatusOK } - if len(manifestBytes) == 0 { - mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) - if err != nil { - return job.Error(err) - } - js, err := libtrust.NewJSONSignature(mBytes) - if err != nil { - return job.Error(err) - } - - if err = js.Sign(s.trustKey); err != nil { - return job.Error(err) - } - - signedBody, err := js.PrettySignature("signatures") - if err != nil { - return job.Error(err) - } - log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) - - manifestBytes = string(signedBody) - } - - manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) - if err != nil { - return job.Errorf("error verifying manifest: %s", err) - } - - if err := checkValidManifest(manifest); err != nil { - return job.Errorf("invalid manifest: %s", err) - } + // error out, no fallback to V1 + return job.Error(err) + } - if !verified { - log.Debugf("Pushing unverified image") + if err != nil { + reposLen := 1 + if tag == "" { + reposLen = len(s.Repositories[repoInfo.LocalName]) } - - for i := len(manifest.FSLayers) - 1; i >= 0; i-- { - var ( - sumStr = manifest.FSLayers[i].BlobSum - imgJSON = []byte(manifest.History[i].V1Compatibility) - ) - - sumParts := strings.SplitN(sumStr, ":", 2) - if len(sumParts) < 2 { - return job.Errorf("Invalid checksum: %s", sumStr) - } - manifestSum := sumParts[1] - - img, err := image.NewImgJSON(imgJSON) - if err != nil { - return job.Errorf("Failed to parse json: %s", err) - } - - img, err = s.graph.Get(img.ID) - if err != nil { + job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) + // If it fails, try to get the repository + if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { + if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { return job.Error(err) } - - arch, err := img.TarLayer() - if err != nil { - return job.Errorf("Could not get tar layer: %s", err) - } - - // Call mount blob - exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) - if err != nil { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) - return job.Error(err) - } - if !exists { - err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) - if err != nil { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) - return job.Error(err) - } - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) - } else { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) - } - } - - // push the manifest - err = r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) - if err != nil { - return job.Error(err) - } - - // done, no fallback to V1 - return engine.StatusOK - } else { - if err != nil { - reposLen := 1 - if tag == "" { - reposLen = len(s.Repositories[repoInfo.LocalName]) - } - job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) - // If it fails, try to get the repository - if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { - if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { - return job.Error(err) - } - return engine.StatusOK - } - return job.Error(err) + return engine.StatusOK } + return job.Error(err) + } - var token []string - job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) - if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { - return job.Error(err) - } - return engine.StatusOK + var token []string + job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) + if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { + return job.Error(err) } + return engine.StatusOK } From 43e1d96190eaf9a3d04919bdc36d2e60235de067 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Thu, 15 Jan 2015 12:23:21 -0800 Subject: [PATCH 26/26] Identify push operation by command http header To allow remotes to understand the operation being carried out during an API request to the registry, we've added a header indicating the engine command. Mostly, this is advisory but a registry may take action based on the field. This changeset only adds this for the "push" command. Signed-off-by: Stephen J Day --- graph/push.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/graph/push.go b/graph/push.go index 32f1b5359867c..139e0487f2909 100644 --- a/graph/push.go +++ b/graph/push.go @@ -313,6 +313,10 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) + // Set a header so remotes can identify the command being carried out. If + // present, the remote may act on the field but this is mostly advisory. + metaHeaders["Docker-Command"] = "push" + if _, err := s.poolAdd("push", repoInfo.LocalName); err != nil { return job.Error(err) }