Browse Source

Refactor image struct with lessons learned.

Remove ACI support
layerset
Till Wegmueller 3 years ago
parent
commit
9871518014
  1. 216
      aciutil/flags.go
  2. 18
      aciutil/labels.go
  3. 23
      aciutil/utils.go
  4. 9
      aciutil/values.go
  5. 2
      cmd/imageadm/create.go
  6. 9
      cmd/imageadm/import.go
  7. 6
      cmd/imageadm/list.go
  8. 98
      common/common.go
  9. 7
      common/errors.go
  10. 68
      common/utils.go
  11. 150
      dist/authchallenge.go
  12. 23
      dist/basictransport.go
  13. 108
      dist/blob.go
  14. 120
      dist/client.go
  15. 60
      dist/error.go
  16. 66
      dist/json.go
  17. 126
      dist/manifest.go
  18. 26
      dist/repositories.go
  19. 25
      dist/tags.go
  20. 130
      dist/tokentransport.go
  21. 91
      host/build.go
  22. 2
      host/container.go
  23. 37
      host/image.go
  24. 20
      host/interface.go
  25. 2
      image/build/config_test.go
  26. 11
      image/clone.go
  27. 46
      image/destroy.go
  28. 131
      image/discovery/aci.go
  29. 15
      image/find.go
  30. 40
      image/global.go
  31. 239
      image/image.go
  32. 604
      image/imagefile/config.go
  33. 112
      image/imagefile/config_test.go
  34. 15
      image/import.go
  35. 19
      image/importzone.go
  36. 41
      image/layer.go
  37. 49
      image/list.go
  38. 39
      image/meta.go
  39. 40
      image/oci.go
  40. 2
      image/oci/metadata_compat.go
  41. 17
      image/oci/reader.go
  42. 19
      image/oci/writer.go
  43. 86
      image/reference/reference.go
  44. 24
      image/reference/reference_test.go
  45. 98
      image/repository.go
  46. 18
      image/repository_test.go
  47. 9
      image/transport/aci.go
  48. 4
      image/transport/docker_test.go
  49. 82
      image/transport/fetch.go
  50. 98
      image/unpack.go
  51. 66
      image/update.go
  52. 59
      image/volume.go
  53. 19
      ips/command.go
  54. 32
      ips/image.go
  55. 390
      keystore/keystore.go
  56. 185
      keystore/keystore_test.go
  57. 131
      keystore/keystoretest/keygen/keygen.go
  58. 537
      keystore/keystoretest/keymap.go
  59. 50
      keystore/keystoretest/openpgp.go
  60. 85
      pod/container.go
  61. 29
      pod/manifest.go
  62. 4
      pod/options.go

216
aciutil/flags.go

@ -1,216 +0,0 @@
package aciutil
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io/ioutil"
"os"
"path"
"runtime"
"strconv"
"strings"
"github.com/appc/spec/discovery"
"github.com/appc/spec/schema"
"github.com/appc/spec/schema/types"
)
type AnnotationsFlag types.Annotations
func (afl *AnnotationsFlag) String() string {
vv := make([]string, len(*afl))
for i, ann := range *afl {
vv[i] = fmt.Sprintf("%v=%#v", ann.Name, ann.Value)
}
return fmt.Sprintf("[%v]", strings.Join(vv, ","))
}
func (afl *AnnotationsFlag) Set(val string) error {
pieces := strings.SplitN(val, "=", 2)
if len(pieces) != 2 {
return errors.New("Annotations must be provided in NAME=VALUE format")
} else if name, err := types.NewACIdentifier(pieces[0]); err != nil {
return err
} else {
(*types.Annotations)(afl).Set(*name, pieces[1])
return nil
}
}
type ExposedPortsFlag []types.ExposedPort
func (epfl *ExposedPortsFlag) String() string {
vv := make([]string, len(*epfl))
for i, ep := range *epfl {
if ep.HostPort == 0 {
vv[i] = ep.Name.String()
} else {
vv[i] = fmt.Sprintf("%v=%v", ep.Name, ep.HostPort)
}
}
return fmt.Sprintf("[%v]", strings.Join(vv, ","))
}
func (epfl *ExposedPortsFlag) Set(val string) error {
ep := types.ExposedPort{}
pieces := strings.SplitN(val, "=", 2)
if name, err := types.NewACName(pieces[0]); err != nil {
return err
} else {
ep.Name = *name
}
if len(pieces) == 2 {
if hp, err := strconv.ParseUint(pieces[1], 10, 0); err != nil {
return err
} else {
ep.HostPort = uint(hp)
}
}
// TODO: check for duplicates? Or do we validate that later (by
// serializing & reparsing JSON)?
*epfl = append(*epfl, ep)
return nil
}
type VolumesFlag []types.Volume
func (vfl *VolumesFlag) String() string {
return fmt.Sprint(([]types.Volume)(*vfl))
}
func (vfl *VolumesFlag) Set(val string) error {
if !strings.ContainsRune(val, ',') {
if pieces := strings.SplitN(val, ":", 2); len(pieces) == 1 {
val += ",kind=empty"
} else {
val = fmt.Sprintf("%v,kind=host,source=%v", pieces[0], pieces[1])
}
}
if val[0] == '-' {
val = val[1:] + ",readOnly=true"
}
if v, err := types.VolumeFromString(val); err != nil {
return err
} else {
// TODO: check for duplicates?
*vfl = append(*vfl, *v)
return nil
}
}
type PodManifestJSONFlag schema.PodManifest
func (pmjf *PodManifestJSONFlag) String() string {
return "[PATH]"
}
func readFileOrStdin(path string) ([]byte, error) {
if path == "-" {
return ioutil.ReadAll(os.Stdin)
}
return ioutil.ReadFile(path)
}
func (pmjf *PodManifestJSONFlag) Set(val string) error {
if bb, err := readFileOrStdin(val); err != nil {
return err
} else if err := json.Unmarshal(bb, (*schema.PodManifest)(pmjf)); err != nil {
return err
} else {
return nil
}
}
type MountsFlag []schema.Mount
func (mf *MountsFlag) String() string {
return fmt.Sprint(*mf)
}
func (mf *MountsFlag) Set(val string) error {
mnt := schema.Mount{}
pieces := strings.SplitN(val, ":", 2)
if name, err := types.NewACName(pieces[0]); err != nil {
return err
} else {
mnt.Volume = *name
}
if len(pieces) == 1 {
mnt.Path = mnt.Volume.String()
} else {
mnt.Path = pieces[1]
}
*mf = append(*mf, mnt)
return nil
}
func PodManifestFlags(fl *flag.FlagSet, pm *schema.PodManifest) {
fl.Var((*PodManifestJSONFlag)(pm), "f", "Read JSON pod manifest file")
fl.Var((*AnnotationsFlag)(&pm.Annotations), "a", "Add annotation (NAME=VALUE)")
fl.Var((*ExposedPortsFlag)(&pm.Ports), "p", "Expose port (NAME[=HOST_PORT])")
fl.Var((*VolumesFlag)(&pm.Volumes), "v", "Define volume")
}
func ParseImageName(name string) (types.ACIdentifier, types.Labels, error) {
app, err := discovery.NewAppFromString(name)
if err != nil {
return "", nil, err
}
if app.Labels["os"] == "" {
app.Labels["os"] = runtime.GOOS
}
if app.Labels["arch"] == "" {
app.Labels["arch"] = runtime.GOARCH
}
labels, err := types.LabelsFromMap(app.Labels)
if err != nil {
return "", nil, err
}
return app.Name, labels, nil
}
func parseApp(args []string) ([]string, *schema.RuntimeApp, error) {
if len(args) == 0 {
return nil, nil, nil
}
rtapp := schema.RuntimeApp{}
// Parse first argument (image name)
if h, err := types.NewHash(args[0]); err == nil {
rtapp.Image.ID = *h
rtapp.Name.Set(h.String()) // won't err
} else if name, labels, err := ParseImageName(args[0]); err == nil {
rtapp.Image.Name = &name
rtapp.Name.Set(path.Base(name.String())) // won't err here
rtapp.Image.Labels = labels
} else {
return args, nil, err
}
fl := flag.NewFlagSet(args[0], flag.ExitOnError)
fl.Var(&rtapp.Name, "name", "App name")
fl.Var((*AnnotationsFlag)(&rtapp.Annotations), "a", "Add annotation (NAME=VALUE)")
fl.Var((*MountsFlag)(&rtapp.Mounts), "m", "Mount volume (VOLUME[:MOUNTPOINT])")
// TODO: app override
fl.Parse(args[1:])
return fl.Args(), &rtapp, nil
}
func ParseApps(pm *schema.PodManifest, args []string) error {
for len(args) > 0 {
if rest, rtapp, err := parseApp(args); err != nil {
return err
} else {
pm.Apps = append(pm.Apps, *rtapp)
args = rest
}
}
return nil
}

18
aciutil/labels.go

@ -1,18 +0,0 @@
package aciutil
import "github.com/appc/spec/schema/types"
// True if all labels in `labels` are present in `candidate` and have
// the same value
func MatchLabels(labels, candidate types.Labels) bool {
if len(labels) == 0 {
return true
}
cmap := candidate.ToMap()
for _, label := range labels {
if v, ok := cmap[label.Name]; !ok || v != label.Value {
return false
}
}
return true
}

23
aciutil/utils.go

@ -1,23 +0,0 @@
package aciutil
import (
"crypto/sha512"
"github.com/appc/spec/schema"
"github.com/appc/spec/schema/types"
)
func IsHashPartial(hash *types.Hash) bool {
// We assume that hash.typ == "sha512". The field is not exported
// (WHY?), so we can't double-check that.
return len(hash.Val) < sha512.Size*2
}
func IsPodManifestEmpty(pm *schema.PodManifest) bool {
return pm == nil ||
(len(pm.Apps) == 0 &&
len(pm.Volumes) == 0 &&
len(pm.Isolators) == 0 &&
len(pm.Annotations) == 0 &&
len(pm.Ports) == 0)
}

9
aciutil/values.go

@ -1,9 +0,0 @@
package aciutil
import "github.com/appc/spec/schema/types"
// Empty ACName
const ACNoName = types.ACName("")
// Empty ACIdentifier
const ACNoIdentifier = types.ACIdentifier("")

2
cmd/imageadm/create.go

@ -18,7 +18,7 @@ var createCmd = &cobra.Command{
if err != nil {
return tracerr.Wrap(err)
}
fmt.Printf("cloned image %s into %s\n", baseRef, img.Reference)
fmt.Printf("cloned image %s into %s\n", baseRef, img.Name)
return nil
},
}

9
cmd/imageadm/import.go

@ -20,16 +20,13 @@ var importCmd = &cobra.Command{
return fmt.Errorf("only zone Import has been implemented yet")
}
ref := reference.NewReferenceString(args[0])
ds, err := h.Dataset.GetChildDataset("images")
if err != nil {
return tracerr.Wrap(err)
}
img, err := image.CreateEmptyImage(ref, ds)
img, err := image.CreateEmptyRepository(ref)
if err != nil {
return tracerr.Wrap(err)
}
return img.ImportZone(zoneName)
return img.ImportZone(ref, zoneName)
},
}

6
cmd/imageadm/list.go

@ -12,11 +12,11 @@ var listCmd = &cobra.Command{
Short: "List all installed images",
Long: `List all images that are currently installed`,
Run: func(cmd *cobra.Command, args []string) {
images := h.Images()
images := h.Repositories()
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Name", "Tags", "Size"})
table.SetHeader([]string{"Name", "Repositories", "Size"})
for _, img := range images {
table.Append([]string{img.Name, img.Tags.String(), img.GetSizeString()})
table.Append([]string{img.Name, img.Images.String(), img.GetSizeString()})
}
table.Render()
},

98
common/common.go

@ -1,98 +0,0 @@
package common
import (
"errors"
"fmt"
"log"
"os"
)
const (
sharedVolumesDir = "/sharedVolumes"
SharedVolumePerm = os.FileMode(0755)
stage1Dir = "/stage1"
stage2Dir = "/opt/stage2"
AppsInfoDir = "/appsinfo"
EnvLockFd = "POD_LOCK_FD"
Stage1TreeStoreIDFilename = "stage1TreeStoreID"
AppTreeStoreIDFilename = "treeStoreID"
OverlayPreparedFilename = "overlay-prepared"
PrivateUsersPreparedFilename = "private-users-prepared"
PrepareLock = "prepareLock"
MetadataServicePort = 18112
MetadataServiceRegSock = "/run/pod/metadata-svc.sock"
APIServiceListenAddr = "localhost:15441"
DefaultLocalConfigDir = "/etc/pod"
DefaultSystemConfigDir = "/lib/pod"
// Default perm bits for the regular files
// within the stage1 directory. (e.g. image manifest,
// pod manifest, stage1ID, etc).
DefaultRegularFilePerm = os.FileMode(0640)
// Default perm bits for the regular directories
// within the stage1 directory.
DefaultRegularDirPerm = os.FileMode(0750)
)
var (
Errlog *log.Logger
Stdlog *log.Logger
)
func NotSupportedError(functionality string) error {
return errors.New(fmt.Sprintf("Functionality %s is currently not Supported", functionality))
}
func InvalidConfiguration(section string) error {
return errors.New(fmt.Sprintf("Invalid Configuration %s is not correct please fix", section))
}
func init() {
Stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
Errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}
func RemoveDuplicates(xs *[]string) {
found := make(map[string]bool)
j := 0
for i, x := range *xs {
if !found[x] {
found[x] = true
(*xs)[j] = (*xs)[i]
j++
}
}
*xs = (*xs)[:j]
}
// Exists reports whether the named file or directory exists.
func FileExists(path string) bool {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}
func RemoveEmpties(xs *[]string) {
j := 0
for i, x := range *xs {
if x != "" {
(*xs)[j] = (*xs)[i]
j++
}
}
*xs = (*xs)[:j]
}
func ExitWithErr(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, format+"\n", values...)
os.Exit(1)
}

7
common/errors.go

@ -1,7 +0,0 @@
package common
import "fmt"
var ErrNotFound = fmt.Errorf("not found")
var ErrUsage = fmt.Errorf("wrong usage of command")

68
common/utils.go

@ -1,68 +0,0 @@
package common
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
"github.com/appc/spec/aci"
"github.com/appc/spec/schema/types"
"github.com/spf13/viper"
)
const version = "0.0.1"
func Version() string {
if revision := viper.GetString("version.git"); revision != "" {
return fmt.Sprintf("%v+git(%v)", version, revision)
} else {
return version
}
}
func ConsoleApp(username string) *types.App {
return &types.App{
Exec: []string{"/usr/bin/login", "-fp", username},
User: "root",
}
}
// FIXME: mostly copy/paste from github.com/appc/spec/actool/validate.go
func DecompressingReader(rd io.Reader) (io.Reader, error) {
brd := bufio.NewReaderSize(rd, 1024)
header, err := brd.Peek(768)
if err != nil {
return nil, err
}
typ, err := aci.DetectFileType(bytes.NewReader(header))
if err != nil {
return nil, err
}
var r io.Reader
switch typ {
case aci.TypeGzip:
r, err = gzip.NewReader(brd)
if err != nil {
return nil, err
}
case aci.TypeBzip2:
r = bzip2.NewReader(brd)
case aci.TypeXz:
r, err = aci.NewXzReader(brd)
if err != nil {
return nil, err
}
case aci.TypeTar:
r = brd
case aci.TypeUnknown:
return nil, fmt.Errorf("unknown filetype")
default:
panic(fmt.Sprintf("bad type returned from DetectFileType: %v", typ))
}
return r, nil
}

150
dist/authchallenge.go vendored

@ -0,0 +1,150 @@
package dist
import (
"net/http"
"strings"
)
// Octet types from RFC 2616.
type octetType byte
// AuthorizationChallenge carries information
// from a WWW-Authenticate response header.
type AuthorizationChallenge struct {
Scheme string
Parameters map[string]string
}
var octetTypes [256]octetType
const (
isToken octetType = 1 << iota
isSpace
)
func init() {
// OCTET = <any 8-bit sequence of data>
// CHAR = <any US-ASCII character (octets 0 - 127)>
// CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
// CR = <US-ASCII CR, carriage return (13)>
// LF = <US-ASCII LF, linefeed (10)>
// SP = <US-ASCII SP, space (32)>
// HT = <US-ASCII HT, horizontal-tab (9)>
// <"> = <US-ASCII double-quote mark (34)>
// CRLF = CR LF
// LWS = [CRLF] 1*( SP | HT )
// TEXT = <any OCTET except CTLs, but including LWS>
// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
// | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {
t |= isToken
}
octetTypes[c] = t
}
}
func parseAuthHeader(header http.Header) []*AuthorizationChallenge {
var challenges []*AuthorizationChallenge
for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] {
v, p := parseValueAndParams(h)
if v != "" {
challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p})
}
}
return challenges
}
func parseValueAndParams(header string) (value string, params map[string]string) {
params = make(map[string]string)
value, s := expectToken(header)
if value == "" {
return
}
value = strings.ToLower(value)
s = "," + skipSpace(s)
for strings.HasPrefix(s, ",") {
var pkey string
pkey, s = expectToken(skipSpace(s[1:]))
if pkey == "" {
return
}
if !strings.HasPrefix(s, "=") {
return
}
var pvalue string
pvalue, s = expectTokenOrQuoted(s[1:])
if pvalue == "" {
return
}
pkey = strings.ToLower(pkey)
params[pkey] = pvalue
s = skipSpace(s)
}
return
}
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isSpace == 0 {
break
}
}
return s[i:]
}
func expectToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isToken == 0 {
break
}
}
return s[:i], s[i:]
}
func expectTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return expectToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + i; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}

23
dist/basictransport.go vendored

@ -0,0 +1,23 @@
package dist
import (
"net/http"
"strings"
)
type BasicTransport struct {
Transport http.RoundTripper
URL string
Username string
Password string
}
func (t *BasicTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if strings.HasPrefix(req.URL.String(), t.URL) {
if t.Username != "" || t.Password != "" {
req.SetBasicAuth(t.Username, t.Password)
}
}
resp, err := t.Transport.RoundTrip(req)
return resp, err
}

108
dist/blob.go vendored

@ -0,0 +1,108 @@
package dist
import (
"io"
"net/http"
"net/url"
"github.com/docker/distribution"
"github.com/opencontainers/go-digest"
)
func (c *Client) DownloadBlob(repository string, digest digest.Digest) (io.ReadCloser, error) {
url := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.download url=%s repository=%s digest=%s", url, repository, digest)
resp, err := c.httpClient.Get(url)
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (c *Client) UploadBlob(repository string, digest digest.Digest, content io.Reader) error {
uploadUrl, err := c.initiateUpload(repository)
if err != nil {
return err
}
q := uploadUrl.Query()
q.Set("digest", digest.String())
uploadUrl.RawQuery = q.Encode()
c.Logf("registry.blob.upload url=%s repository=%s digest=%s", uploadUrl, repository, digest)
upload, err := http.NewRequest("PUT", uploadUrl.String(), content)
if err != nil {
return err
}
upload.Header.Set("Content-Type", "application/octet-stream")
_, err = c.httpClient.Do(upload)
return err
}
func (c *Client) HasBlob(repository string, digest digest.Digest) (bool, error) {
checkUrl := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.check url=%s repository=%s digest=%s", checkUrl, repository, digest)
resp, err := c.httpClient.Head(checkUrl)
if resp != nil {
defer resp.Body.Close()
}
if err == nil {
return resp.StatusCode == http.StatusOK, nil
}
urlErr, ok := err.(*url.Error)
if !ok {
return false, err
}
httpErr, ok := urlErr.Err.(*ErrorResponse)
if !ok {
return false, err
}
if httpErr.HTTPStatusCode == http.StatusNotFound {
return false, nil
}
return false, err
}
func (c *Client) BlobMetadata(repository string, digest digest.Digest) (distribution.Descriptor, error) {
checkUrl := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.check url=%s repository=%s digest=%s", checkUrl, repository, digest)
resp, err := c.httpClient.Head(checkUrl)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return distribution.Descriptor{}, err
}
return distribution.Descriptor{
Digest: digest,
Size: resp.ContentLength,
}, nil
}
func (c *Client) initiateUpload(repository string) (*url.URL, error) {
initiateUrl := c.url("/v2/%s/blobs/uploads/", repository)
c.Logf("registry.blob.initiate-upload url=%s repository=%s", initiateUrl, repository)
resp, err := c.httpClient.Post(initiateUrl, "application/octet-stream", nil)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return nil, err
}
location := resp.Header.Get("Location")
locationUrl, err := url.Parse(location)
if err != nil {
return nil, err
}
return locationUrl, nil
}

120
dist/client.go vendored

@ -0,0 +1,120 @@
package dist
import (
"crypto/tls"
"fmt"
"log"
"net/http"
"strings"
)
type Client struct {
httpClient *http.Client
URL string
Logf LogfCallback
}
type LogfCallback func(format string, args ...interface{})
/*
* Discard log messages silently.
*/
func Quiet(format string, args ...interface{}) {
/* discard logs */
}
/*
* Pass log messages along to Go's "log" module.
*/
func Log(format string, args ...interface{}) {
log.Printf(format, args...)
}
/*
* Create a new Registry with the given URL and credentials, then Ping()s it
* before returning it to verify that the registry is available.
*
* You can, alternately, construct a Registry manually by populating the fields.
* This passes http.DefaultTransport to WrapTransport when creating the
* http.Client.
*/
func NewClient(registryUrl, username, password string) (*Client, error) {
transport := http.DefaultTransport
return newFromTransport(registryUrl, username, password, transport, Log)
}
/*
* Create a new Registry, as with New, using an http.Transport that disables
* SSL certificate verification.
*/
func NewInsecureClient(registryUrl, username, password string) (*Client, error) {
transport := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
return newFromTransport(registryUrl, username, password, transport, Log)
}
/*
* Given an existing http.RoundTripper such as http.DefaultTransport, build the
* transport stack necessary to authenticate to the Docker registry API. This
* adds in support for OAuth bearer tokens and HTTP Basic auth, and sets up
* error handling this library relies on.
*/
func WrapTransport(transport http.RoundTripper, url, username, password string) http.RoundTripper {
tokenTransport := &TokenTransport{
Transport: transport,
Username: username,
Password: password,
}
basicAuthTransport := &BasicTransport{
Transport: tokenTransport,
URL: url,
Username: username,
Password: password,
}
errorTransport := &ErrorTransport{
Transport: basicAuthTransport,
}
return errorTransport
}
func newFromTransport(registryUrl, username, password string, transport http.RoundTripper, logf LogfCallback) (*Client, error) {
url := strings.TrimSuffix(registryUrl, "/")
transport = WrapTransport(transport, url, username, password)
registry := &Client{
URL: url,
httpClient: &http.Client{
Transport: transport,
},
Logf: logf,
}
if err := registry.Ping(); err != nil {
return nil, err
}
return registry, nil
}
func (c *Client) url(pathTemplate string, args ...interface{}) string {
pathSuffix := fmt.Sprintf(pathTemplate, args...)
url := fmt.Sprintf("%s%s", c.URL, pathSuffix)
return url
}
func (c *Client) Ping() error {
url := c.url("/v2/")
c.Logf("registry.ping url=%s", url)
resp, err := c.httpClient.Get(url)
if err == nil {
defer resp.Body.Close()
return nil
}
return err
}

60
dist/error.go vendored

@ -0,0 +1,60 @@
package dist
import (
"encoding/json"
"fmt"
"net/http"
)
type ErrorResponse struct {
HTTPStatusCode int
Errors []RegistryError `json:"errors"`
}
func (e ErrorResponse) Error() string {
errors := ""
for _, err := range e.Errors {
errors += err.Error() + "\n"
}
return errors
}
type RegistryError struct {
Code string `json:"code"`
Message string `json:"message"`
Detail json.RawMessage `json:"detail"`
}
func (r RegistryError) Error() string {
return fmt.Sprintf("%s: %s", r.Code, r.Message)
}
func (r RegistryError) Details() string {
return string(r.Detail)
}
type ErrorTransport struct {
Transport http.RoundTripper
}
func (t *ErrorTransport) RoundTrip(request *http.Request) (*http.Response, error) {
resp, err := t.Transport.RoundTrip(request)
if err != nil {
return resp, err
}
if resp.StatusCode >= 400 {
defer resp.Body.Close()
ErrResp := ErrorResponse{}
if err := json.NewDecoder(resp.Body).Decode(&ErrResp); err != nil {
return nil, fmt.Errorf("http: failed to read response body (status=%v, err=%q)", resp.StatusCode, err)
}
ErrResp.HTTPStatusCode = resp.StatusCode
return nil, &ErrResp
}
return resp, err
}

66
dist/json.go vendored

@ -0,0 +1,66 @@
package dist
import (
"encoding/json"
"errors"
"net/http"
"regexp"
)
var (
ErrNoMorePages = errors.New("No more pages")
)
func (c *Client) getJson(url string, response interface{}) error {
resp, err := c.httpClient.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return err
}
return nil
}
// getPaginatedJson accepts a string and a pointer, and returns the
// next page URL while updating pointed-to variable with a parsed JSON
// value. When there are no more pages it returns `ErrNoMorePages`.
func (c *Client) getPaginatedJson(url string, response interface{}) (string, error) {
resp, err := c.httpClient.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return "", err
}
return getNextLink(resp)
}
// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5)
// Link header. For example,
//
// <http://registry.example.com/v2/_catalog?n=5&last=tag5>; type="application/json"; rel="next"
//
// The URL is _supposed_ to be wrapped by angle brackets `< ... >`,
// but e.g., quay.io does not include them. Similarly, params like
// `rel="next"` may not have quoted values in the wild.
var nextLinkRE = regexp.MustCompile(`^ *<?([^;>]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`)
func getNextLink(resp *http.Response) (string, error) {
for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
parts := nextLinkRE.FindStringSubmatch(link)
if parts != nil {
return parts[1], nil
}
}
return "", ErrNoMorePages
}

126
dist/manifest.go vendored

@ -0,0 +1,126 @@
package dist
import (
"bytes"
"io/ioutil"
"net/http"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/opencontainers/go-digest"
)
func (c *Client) Manifest(repository, reference string) (*schema1.SignedManifest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", schema1.MediaTypeManifest)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
signedManifest := &schema1.SignedManifest{}
err = signedManifest.UnmarshalJSON(body)
if err != nil {
return nil, err
}
return signedManifest, nil
}
func (c *Client) ManifestV2(repository, reference string) (*schema2.DeserializedManifest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", schema2.MediaTypeManifest)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
deserialized := &schema2.DeserializedManifest{}
err = deserialized.UnmarshalJSON(body)
if err != nil {
return nil, err
}
return deserialized, nil
}
func (c *Client) ManifestDigest(repository, reference string) (digest.Digest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
resp, err := c.httpClient.Head(url)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return "", err
}
return digest.Parse(resp.Header.Get("Docker-Content-Digest"))
}
func (c *Client) DeleteManifest(repository string, digest digest.Digest) error {
url := c.url("/v2/%s/manifests/%s", repository, digest)
c.Logf("registry.manifest.delete url=%s repository=%s reference=%s", url, repository, digest)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return err
}
return nil
}
func (c *Client) PutManifest(repository, reference string, manifest distribution.Manifest) error {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)
mediaType, payload, err := manifest.Payload()
if err != nil {
return err
}
buffer := bytes.NewBuffer(payload)
req, err := http.NewRequest("PUT", url, buffer)
if err != nil {
return err
}
req.Header.Set("Content-Type", mediaType)
resp, err := c.httpClient.Do(req)
if resp != nil {
defer resp.Body.Close()
}
return err
}

26
dist/repositories.go vendored

@ -0,0 +1,26 @@
package dist
type repositoriesResponse struct {
Repositories []string `json:"repositories"`
}
func (c *Client) Repositories() ([]string, error) {
url := c.url("/v2/_catalog")
repos := make([]string, 0, 10)
var err error //We create this here, otherwise url will be rescoped with :=
var response repositoriesResponse
for {
c.Logf("registry.repositories url=%s", url)
url, err = c.getPaginatedJson(url, &response)
switch err {
case ErrNoMorePages:
repos = append(repos, response.Repositories...)
return repos, nil
case nil:
repos = append(repos, response.Repositories...)
continue
default:
return nil, err
}
}
}

25
dist/tags.go vendored

@ -0,0 +1,25 @@
package dist
type tagsResponse struct {
Tags []string `json:"tags"`
}
func (c *Client) Tags(repository string) (tags []string, err error) {
url := c.url("/v2/%s/tags/list", repository)
var response tagsResponse
for {
c.Logf("registry.tags url=%s repository=%s", url, repository)
url, err = c.getPaginatedJson(url, &response)
switch err {
case ErrNoMorePages:
tags = append(tags, response.Tags...)
return tags, nil
case nil:
tags = append(tags, response.Tags...)
continue
default:
return nil, err
}
}
}

130
dist/tokentransport.go vendored

@ -0,0 +1,130 @@
package dist
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
)
type TokenTransport struct {
Transport http.RoundTripper
Username string
Password string
}
func (t *TokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := t.Transport.RoundTrip(req)
if err != nil {
return resp, err
}
if authService := isTokenDemand(resp); authService != nil {
resp, err = t.authAndRetry(authService, req)
}
return resp, err
}
type authToken struct {
Token string `json:"token"`
}
func (t *TokenTransport) authAndRetry(authService *authService, req *http.Request) (*http.Response, error) {
token, authResp, err := t.auth(authService)
if err != nil {
return authResp, err
}
retryResp, err := t.retry(req, token)
return retryResp, err
}
func (t *TokenTransport) auth(authService *authService) (string, *http.Response, error) {
authReq, err := authService.Request(t.Username, t.Password)
if err != nil {
return "", nil, err
}
client := http.Client{
Transport: t.Transport,
}
response, err := client.Do(authReq)
if err != nil {
return "", nil, err
}
if response.StatusCode != http.StatusOK {
return "", response, err
}
defer response.Body.Close()
var authToken authToken
decoder := json.NewDecoder(response.Body)
err = decoder.Decode(&authToken)
if err != nil {
return "", nil, err
}
return authToken.Token, nil, nil
}
func (t *TokenTransport) retry(req *http.Request, token string) (*http.Response, error) {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
resp, err := t.Transport.RoundTrip(req)
return resp, err
}
type authService struct {
Realm string
Service string
Scope string
}
func (authService *authService) Request(username, password string) (*http.Request, error) {
url, err := url.Parse(authService.Realm)
if err != nil {
return nil, err
}
q := url.Query()
q.Set("service", authService.Service)
if authService.Scope != "" {
q.Set("scope", authService.Scope)
}
url.RawQuery = q.Encode()
request, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return nil, err
}
if username != "" || password != "" {
request.SetBasicAuth(username, password)
}
return request, err
}
func isTokenDemand(resp *http.Response) *authService {
if resp == nil {
return nil
}
if resp.StatusCode != http.StatusUnauthorized {
return nil
}
return parseOauthHeader(resp)
}
func parseOauthHeader(resp *http.Response) *authService {
challenges := parseAuthHeader(resp.Header)
for _, challenge := range challenges {
if challenge.Scheme == "bearer" {
return &authService{
Realm: challenge.Parameters["realm"],
Service: challenge.Parameters["service"],
Scope: challenge.Parameters["scope"],
}
}
}
return nil
}

91
host/build.go

@ -29,7 +29,8 @@ const buildHelperName = "buildhelper"
const helperPath = "/usr/lib/brand/pod/helpers/%s/" + buildHelperName
func (h *Host) RunBuild(cfg *build.ImageConfig, opts *pod.Options) error {
buildImages := make(map[string]*image.Image)
buildImages := make(map[string]*image.Repository)
for _, imgCfg := range cfg.Images {
logrus.Infof("starting image build for %s", imgCfg.Name)
if cfg.Maintainer != nil {
@ -47,20 +48,27 @@ func (h *Host) RunBuild(cfg *build.ImageConfig, opts *pod.Options) error {
for _, op := range imgCfg.CopyOperations {
logrus.Debugf("copying %s from source Image %s", op.Name, op.SourceImage)
srcRefString := reference.NewReferenceString(op.SourceImage)
srcImg, ok := buildImages[srcRefString.String()]
srcRepository, ok := buildImages[srcRefString.String()]
if !ok {
return fmt.Errorf("no image %s has bin built yet", op.SourceImage)
}
digest, ok := srcImg.Tags[