cneira
/
opencloud
forked from OpenCloud/opencloud
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Refactor image struct with lessons learned. 

Remove ACI support
master
Till Wegmueller 4 years ago
parent
9736c44e76
commit
9871518014
62 changed files with 1500 additions and 3387 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 216
      aciutil/flags.go
  2. 18
      aciutil/labels.go
  3. 23
      aciutil/utils.go
  4. 9
      aciutil/values.go
  5. 2
      cmd/imageadm/create.go
  6. 9
      cmd/imageadm/import.go
  7. 6
      cmd/imageadm/list.go
  8. 98
      common/common.go
  9. 7
      common/errors.go
  10. 68
      common/utils.go
  11. 150
      dist/authchallenge.go
  12. 23
      dist/basictransport.go
  13. 108
      dist/blob.go
  14. 120
      dist/client.go
  15. 60
      dist/error.go
  16. 66
      dist/json.go
  17. 126
      dist/manifest.go
  18. 26
      dist/repositories.go
  19. 25
      dist/tags.go
  20. 130
      dist/tokentransport.go
  21. 91
      host/build.go
  22. 2
      host/container.go
  23. 37
      host/image.go
  24. 20
      host/interface.go
  25. 2
      image/build/config_test.go
  26. 11
      image/clone.go
  27. 46
      image/destroy.go
  28. 131
      image/discovery/aci.go
  29. 15
      image/find.go
  30. 40
      image/global.go
  31. 239
      image/image.go
  32. 604
      image/imagefile/config.go
  33. 112
      image/imagefile/config_test.go
  34. 15
      image/import.go
  35. 19
      image/importzone.go
  36. 41
      image/layer.go
  37. 49
      image/list.go
  38. 39
      image/meta.go
  39. 40
      image/oci.go
  40. 2
      image/oci/metadata_compat.go
  41. 17
      image/oci/reader.go
  42. 19
      image/oci/writer.go
  43. 86
      image/reference/reference.go
  44. 24
      image/reference/reference_test.go
  45. 98
      image/repository.go
  46. 18
      image/repository_test.go
  47. 9
      image/transport/aci.go
  48. 4
      image/transport/docker_test.go
  49. 82
      image/transport/fetch.go
  50. 98
      image/unpack.go
  51. 66
      image/update.go
  52. 59
      image/volume.go
  53. 19
      ips/command.go
  54. 32
      ips/image.go
  55. 390
      keystore/keystore.go
  56. 185
      keystore/keystore_test.go
  57. 131
      keystore/keystoretest/keygen/keygen.go
  58. 537
      keystore/keystoretest/keymap.go
  59. 50
      keystore/keystoretest/openpgp.go
  60. 85
      pod/container.go
  61. 29
      pod/manifest.go
  62. 4
      pod/options.go

216
aciutil/flags.go
Unescape View File

@ -1,216 +0,0 @@
package aciutil
import (
"encoding/json"
"errors"
"flag"
"fmt"
"io/ioutil"
"os"
"path"
"runtime"
"strconv"
"strings"
"github.com/appc/spec/discovery"
"github.com/appc/spec/schema"
"github.com/appc/spec/schema/types"
)
type AnnotationsFlag types.Annotations
func (afl *AnnotationsFlag) String() string {
vv := make([]string, len(*afl))
for i, ann := range *afl {
vv[i] = fmt.Sprintf("%v=%#v", ann.Name, ann.Value)
}
return fmt.Sprintf("[%v]", strings.Join(vv, ","))
}
func (afl *AnnotationsFlag) Set(val string) error {
pieces := strings.SplitN(val, "=", 2)
if len(pieces) != 2 {
return errors.New("Annotations must be provided in NAME=VALUE format")
} else if name, err := types.NewACIdentifier(pieces[0]); err != nil {
return err
} else {
(*types.Annotations)(afl).Set(*name, pieces[1])
return nil
}
}
type ExposedPortsFlag []types.ExposedPort
func (epfl *ExposedPortsFlag) String() string {
vv := make([]string, len(*epfl))
for i, ep := range *epfl {
if ep.HostPort == 0 {
vv[i] = ep.Name.String()
} else {
vv[i] = fmt.Sprintf("%v=%v", ep.Name, ep.HostPort)
}
}
return fmt.Sprintf("[%v]", strings.Join(vv, ","))
}
func (epfl *ExposedPortsFlag) Set(val string) error {
ep := types.ExposedPort{}
pieces := strings.SplitN(val, "=", 2)
if name, err := types.NewACName(pieces[0]); err != nil {
return err
} else {
ep.Name = *name
}
if len(pieces) == 2 {
if hp, err := strconv.ParseUint(pieces[1], 10, 0); err != nil {
return err
} else {
ep.HostPort = uint(hp)
}
}
// TODO: check for duplicates? Or do we validate that later (by
// serializing & reparsing JSON)?
*epfl = append(*epfl, ep)
return nil
}
type VolumesFlag []types.Volume
func (vfl *VolumesFlag) String() string {
return fmt.Sprint(([]types.Volume)(*vfl))
}
func (vfl *VolumesFlag) Set(val string) error {
if !strings.ContainsRune(val, ',') {
if pieces := strings.SplitN(val, ":", 2); len(pieces) == 1 {
val += ",kind=empty"
} else {
val = fmt.Sprintf("%v,kind=host,source=%v", pieces[0], pieces[1])
}
}
if val[0] == '-' {
val = val[1:] + ",readOnly=true"
}
if v, err := types.VolumeFromString(val); err != nil {
return err
} else {
// TODO: check for duplicates?
*vfl = append(*vfl, *v)
return nil
}
}
type PodManifestJSONFlag schema.PodManifest
func (pmjf *PodManifestJSONFlag) String() string {
return "[PATH]"
}
func readFileOrStdin(path string) ([]byte, error) {
if path == "-" {
return ioutil.ReadAll(os.Stdin)
}
return ioutil.ReadFile(path)
}
func (pmjf *PodManifestJSONFlag) Set(val string) error {
if bb, err := readFileOrStdin(val); err != nil {
return err
} else if err := json.Unmarshal(bb, (*schema.PodManifest)(pmjf)); err != nil {
return err
} else {
return nil
}
}
type MountsFlag []schema.Mount
func (mf *MountsFlag) String() string {
return fmt.Sprint(*mf)
}
func (mf *MountsFlag) Set(val string) error {
mnt := schema.Mount{}
pieces := strings.SplitN(val, ":", 2)
if name, err := types.NewACName(pieces[0]); err != nil {
return err
} else {
mnt.Volume = *name
}
if len(pieces) == 1 {
mnt.Path = mnt.Volume.String()
} else {
mnt.Path = pieces[1]
}
*mf = append(*mf, mnt)
return nil
}
func PodManifestFlags(fl *flag.FlagSet, pm *schema.PodManifest) {
fl.Var((*PodManifestJSONFlag)(pm), "f", "Read JSON pod manifest file")
fl.Var((*AnnotationsFlag)(&pm.Annotations), "a", "Add annotation (NAME=VALUE)")
fl.Var((*ExposedPortsFlag)(&pm.Ports), "p", "Expose port (NAME[=HOST_PORT])")
fl.Var((*VolumesFlag)(&pm.Volumes), "v", "Define volume")
}
func ParseImageName(name string) (types.ACIdentifier, types.Labels, error) {
app, err := discovery.NewAppFromString(name)
if err != nil {
return "", nil, err
}
if app.Labels["os"] == "" {
app.Labels["os"] = runtime.GOOS
}
if app.Labels["arch"] == "" {
app.Labels["arch"] = runtime.GOARCH
}
labels, err := types.LabelsFromMap(app.Labels)
if err != nil {
return "", nil, err
}
return app.Name, labels, nil
}
func parseApp(args []string) ([]string, *schema.RuntimeApp, error) {
if len(args) == 0 {
return nil, nil, nil
}
rtapp := schema.RuntimeApp{}
// Parse first argument (image name)
if h, err := types.NewHash(args[0]); err == nil {
rtapp.Image.ID = *h
rtapp.Name.Set(h.String()) // won't err
} else if name, labels, err := ParseImageName(args[0]); err == nil {
rtapp.Image.Name = &name
rtapp.Name.Set(path.Base(name.String())) // won't err here
rtapp.Image.Labels = labels
} else {
return args, nil, err
}
fl := flag.NewFlagSet(args[0], flag.ExitOnError)
fl.Var(&rtapp.Name, "name", "App name")
fl.Var((*AnnotationsFlag)(&rtapp.Annotations), "a", "Add annotation (NAME=VALUE)")
fl.Var((*MountsFlag)(&rtapp.Mounts), "m", "Mount volume (VOLUME[:MOUNTPOINT])")
// TODO: app override
fl.Parse(args[1:])
return fl.Args(), &rtapp, nil
}
func ParseApps(pm *schema.PodManifest, args []string) error {
for len(args) > 0 {
if rest, rtapp, err := parseApp(args); err != nil {
return err
} else {
pm.Apps = append(pm.Apps, *rtapp)
args = rest
}
}
return nil
}

18
aciutil/labels.go
Unescape View File

@ -1,18 +0,0 @@
package aciutil
import "github.com/appc/spec/schema/types"
// True if all labels in `labels` are present in `candidate` and have
// the same value
func MatchLabels(labels, candidate types.Labels) bool {
if len(labels) == 0 {
return true
}
cmap := candidate.ToMap()
for _, label := range labels {
if v, ok := cmap[label.Name]; !ok || v != label.Value {
return false
}
}
return true
}

23
aciutil/utils.go
Unescape View File

@ -1,23 +0,0 @@
package aciutil
import (
"crypto/sha512"
"github.com/appc/spec/schema"
"github.com/appc/spec/schema/types"
)
func IsHashPartial(hash *types.Hash) bool {
// We assume that hash.typ == "sha512". The field is not exported
// (WHY?), so we can't double-check that.
return len(hash.Val) < sha512.Size*2
}
func IsPodManifestEmpty(pm *schema.PodManifest) bool {
return pm == nil ||
(len(pm.Apps) == 0 &&
len(pm.Volumes) == 0 &&
len(pm.Isolators) == 0 &&
len(pm.Annotations) == 0 &&
len(pm.Ports) == 0)
}

9
aciutil/values.go
Unescape View File

@ -1,9 +0,0 @@
package aciutil
import "github.com/appc/spec/schema/types"
// Empty ACName
const ACNoName = types.ACName("")
// Empty ACIdentifier
const ACNoIdentifier = types.ACIdentifier("")

2
cmd/imageadm/create.go
Unescape View File

@ -18,7 +18,7 @@ var createCmd = &cobra.Command{
if err != nil {
return tracerr.Wrap(err)
}
fmt.Printf("cloned image %s into %s\n", baseRef, img.Reference)
fmt.Printf("cloned image %s into %s\n", baseRef, img.Name)
return nil
},
}

9
cmd/imageadm/import.go
Unescape View File

@ -20,16 +20,13 @@ var importCmd = &cobra.Command{
return fmt.Errorf("only zone Import has been implemented yet")
}
ref := reference.NewReferenceString(args[0])
ds, err := h.Dataset.GetChildDataset("images")
if err != nil {
return tracerr.Wrap(err)
}
img, err := image.CreateEmptyImage(ref, ds)
img, err := image.CreateEmptyRepository(ref)
if err != nil {
return tracerr.Wrap(err)
}
return img.ImportZone(zoneName)
return img.ImportZone(ref, zoneName)
},
}

6
cmd/imageadm/list.go
Unescape View File

@ -12,11 +12,11 @@ var listCmd = &cobra.Command{
Short: "List all installed images",
Long: `List all images that are currently installed`,
Run: func(cmd *cobra.Command, args []string) {
images := h.Images()
images := h.Repositories()
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Name", "Tags", "Size"})
table.SetHeader([]string{"Name", "Repositories", "Size"})
for _, img := range images {
table.Append([]string{img.Name, img.Tags.String(), img.GetSizeString()})
table.Append([]string{img.Name, img.Images.String(), img.GetSizeString()})
}
table.Render()
},

98
common/common.go
Unescape View File

@ -1,98 +0,0 @@
package common
import (
"errors"
"fmt"
"log"
"os"
)
const (
sharedVolumesDir = "/sharedVolumes"
SharedVolumePerm = os.FileMode(0755)
stage1Dir = "/stage1"
stage2Dir = "/opt/stage2"
AppsInfoDir = "/appsinfo"
EnvLockFd = "POD_LOCK_FD"
Stage1TreeStoreIDFilename = "stage1TreeStoreID"
AppTreeStoreIDFilename = "treeStoreID"
OverlayPreparedFilename = "overlay-prepared"
PrivateUsersPreparedFilename = "private-users-prepared"
PrepareLock = "prepareLock"
MetadataServicePort = 18112
MetadataServiceRegSock = "/run/pod/metadata-svc.sock"
APIServiceListenAddr = "localhost:15441"
DefaultLocalConfigDir = "/etc/pod"
DefaultSystemConfigDir = "/lib/pod"
// Default perm bits for the regular files
// within the stage1 directory. (e.g. image manifest,
// pod manifest, stage1ID, etc).
DefaultRegularFilePerm = os.FileMode(0640)
// Default perm bits for the regular directories
// within the stage1 directory.
DefaultRegularDirPerm = os.FileMode(0750)
)
var (
Errlog *log.Logger
Stdlog *log.Logger
)
func NotSupportedError(functionality string) error {
return errors.New(fmt.Sprintf("Functionality %s is currently not Supported", functionality))
}
func InvalidConfiguration(section string) error {
return errors.New(fmt.Sprintf("Invalid Configuration %s is not correct please fix", section))
}
func init() {
Stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
Errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
}
func RemoveDuplicates(xs *[]string) {
found := make(map[string]bool)
j := 0
for i, x := range *xs {
if !found[x] {
found[x] = true
(*xs)[j] = (*xs)[i]
j++
}
}
*xs = (*xs)[:j]
}
// Exists reports whether the named file or directory exists.
func FileExists(path string) bool {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}
func RemoveEmpties(xs *[]string) {
j := 0
for i, x := range *xs {
if x != "" {
(*xs)[j] = (*xs)[i]
j++
}
}
*xs = (*xs)[:j]
}
func ExitWithErr(format string, values ...interface{}) {
fmt.Fprintf(os.Stderr, format+"\n", values...)
os.Exit(1)
}

7
common/errors.go
Unescape View File

@ -1,7 +0,0 @@
package common
import "fmt"
var ErrNotFound = fmt.Errorf("not found")
var ErrUsage = fmt.Errorf("wrong usage of command")

68
common/utils.go
Unescape View File

@ -1,68 +0,0 @@
package common
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
"github.com/appc/spec/aci"
"github.com/appc/spec/schema/types"
"github.com/spf13/viper"
)
const version = "0.0.1"
func Version() string {
if revision := viper.GetString("version.git"); revision != "" {
return fmt.Sprintf("%v+git(%v)", version, revision)
} else {
return version
}
}
func ConsoleApp(username string) *types.App {
return &types.App{
Exec: []string{"/usr/bin/login", "-fp", username},
User: "root",
}
}
// FIXME: mostly copy/paste from github.com/appc/spec/actool/validate.go
func DecompressingReader(rd io.Reader) (io.Reader, error) {
brd := bufio.NewReaderSize(rd, 1024)
header, err := brd.Peek(768)
if err != nil {
return nil, err
}
typ, err := aci.DetectFileType(bytes.NewReader(header))
if err != nil {
return nil, err
}
var r io.Reader
switch typ {
case aci.TypeGzip:
r, err = gzip.NewReader(brd)
if err != nil {
return nil, err
}
case aci.TypeBzip2:
r = bzip2.NewReader(brd)
case aci.TypeXz:
r, err = aci.NewXzReader(brd)
if err != nil {
return nil, err
}
case aci.TypeTar:
r = brd
case aci.TypeUnknown:
return nil, fmt.Errorf("unknown filetype")
default:
panic(fmt.Sprintf("bad type returned from DetectFileType: %v", typ))
}
return r, nil
}

150
dist/authchallenge.go vendored
Unescape View File

@ -0,0 +1,150 @@
package dist
import (
"net/http"
"strings"
)
// Octet types from RFC 2616.
type octetType byte
// AuthorizationChallenge carries information
// from a WWW-Authenticate response header.
type AuthorizationChallenge struct {
Scheme string
Parameters map[string]string
}
var octetTypes [256]octetType
const (
isToken octetType = 1 << iota
isSpace
)
func init() {
// OCTET = <any 8-bit sequence of data>
// CHAR = <any US-ASCII character (octets 0 - 127)>
// CTL = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
// CR = <US-ASCII CR, carriage return (13)>
// LF = <US-ASCII LF, linefeed (10)>
// SP = <US-ASCII SP, space (32)>
// HT = <US-ASCII HT, horizontal-tab (9)>
// <"> = <US-ASCII double-quote mark (34)>
// CRLF = CR LF
// LWS = [CRLF] 1*( SP | HT )
// TEXT = <any OCTET except CTLs, but including LWS>
// separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <">
// | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT
// token = 1*<any CHAR except CTLs or separators>
// qdtext = <any TEXT except <">>
for c := 0; c < 256; c++ {
var t octetType
isCtl := c <= 31 || c == 127
isChar := 0 <= c && c <= 127
isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0
if strings.IndexRune(" \t\r\n", rune(c)) >= 0 {
t |= isSpace
}
if isChar && !isCtl && !isSeparator {
t |= isToken
}
octetTypes[c] = t
}
}
func parseAuthHeader(header http.Header) []*AuthorizationChallenge {
var challenges []*AuthorizationChallenge
for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] {
v, p := parseValueAndParams(h)
if v != "" {
challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p})
}
}
return challenges
}
func parseValueAndParams(header string) (value string, params map[string]string) {
params = make(map[string]string)
value, s := expectToken(header)
if value == "" {
return
}
value = strings.ToLower(value)
s = "," + skipSpace(s)
for strings.HasPrefix(s, ",") {
var pkey string
pkey, s = expectToken(skipSpace(s[1:]))
if pkey == "" {
return
}
if !strings.HasPrefix(s, "=") {
return
}
var pvalue string
pvalue, s = expectTokenOrQuoted(s[1:])
if pvalue == "" {
return
}
pkey = strings.ToLower(pkey)
params[pkey] = pvalue
s = skipSpace(s)
}
return
}
func skipSpace(s string) (rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isSpace == 0 {
break
}
}
return s[i:]
}
func expectToken(s string) (token, rest string) {
i := 0
for ; i < len(s); i++ {
if octetTypes[s[i]]&isToken == 0 {
break
}
}
return s[:i], s[i:]
}
func expectTokenOrQuoted(s string) (value string, rest string) {
if !strings.HasPrefix(s, "\"") {
return expectToken(s)
}
s = s[1:]
for i := 0; i < len(s); i++ {
switch s[i] {
case '"':
return s[:i], s[i+1:]
case '\\':
p := make([]byte, len(s)-1)
j := copy(p, s[:i])
escape := true
for i = i + i; i < len(s); i++ {
b := s[i]
switch {
case escape:
escape = false
p[j] = b
j++
case b == '\\':
escape = true
case b == '"':
return string(p[:j]), s[i+1:]
default:
p[j] = b
j++
}
}
return "", ""
}
}
return "", ""
}

23
dist/basictransport.go vendored
Unescape View File

@ -0,0 +1,23 @@
package dist
import (
"net/http"
"strings"
)
type BasicTransport struct {
Transport http.RoundTripper
URL string
Username string
Password string
}
func (t *BasicTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if strings.HasPrefix(req.URL.String(), t.URL) {
if t.Username != "" || t.Password != "" {
req.SetBasicAuth(t.Username, t.Password)
}
}
resp, err := t.Transport.RoundTrip(req)
return resp, err
}

108
dist/blob.go vendored
Unescape View File

@ -0,0 +1,108 @@
package dist
import (
"io"
"net/http"
"net/url"
"github.com/docker/distribution"
"github.com/opencontainers/go-digest"
)
func (c *Client) DownloadBlob(repository string, digest digest.Digest) (io.ReadCloser, error) {
url := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.download url=%s repository=%s digest=%s", url, repository, digest)
resp, err := c.httpClient.Get(url)
if err != nil {
return nil, err
}
return resp.Body, nil
}
func (c *Client) UploadBlob(repository string, digest digest.Digest, content io.Reader) error {
uploadUrl, err := c.initiateUpload(repository)
if err != nil {
return err
}
q := uploadUrl.Query()
q.Set("digest", digest.String())
uploadUrl.RawQuery = q.Encode()
c.Logf("registry.blob.upload url=%s repository=%s digest=%s", uploadUrl, repository, digest)
upload, err := http.NewRequest("PUT", uploadUrl.String(), content)
if err != nil {
return err
}
upload.Header.Set("Content-Type", "application/octet-stream")
_, err = c.httpClient.Do(upload)
return err
}
func (c *Client) HasBlob(repository string, digest digest.Digest) (bool, error) {
checkUrl := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.check url=%s repository=%s digest=%s", checkUrl, repository, digest)
resp, err := c.httpClient.Head(checkUrl)
if resp != nil {
defer resp.Body.Close()
}
if err == nil {
return resp.StatusCode == http.StatusOK, nil
}
urlErr, ok := err.(*url.Error)
if !ok {
return false, err
}
httpErr, ok := urlErr.Err.(*ErrorResponse)
if !ok {
return false, err
}
if httpErr.HTTPStatusCode == http.StatusNotFound {
return false, nil
}
return false, err
}
func (c *Client) BlobMetadata(repository string, digest digest.Digest) (distribution.Descriptor, error) {
checkUrl := c.url("/v2/%s/blobs/%s", repository, digest)
c.Logf("registry.blob.check url=%s repository=%s digest=%s", checkUrl, repository, digest)
resp, err := c.httpClient.Head(checkUrl)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return distribution.Descriptor{}, err
}
return distribution.Descriptor{
Digest: digest,
Size: resp.ContentLength,
}, nil
}
func (c *Client) initiateUpload(repository string) (*url.URL, error) {
initiateUrl := c.url("/v2/%s/blobs/uploads/", repository)
c.Logf("registry.blob.initiate-upload url=%s repository=%s", initiateUrl, repository)
resp, err := c.httpClient.Post(initiateUrl, "application/octet-stream", nil)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return nil, err
}
location := resp.Header.Get("Location")
locationUrl, err := url.Parse(location)
if err != nil {
return nil, err
}
return locationUrl, nil
}

120
dist/client.go vendored
Unescape View File

@ -0,0 +1,120 @@
package dist
import (
"crypto/tls"
"fmt"
"log"
"net/http"
"strings"
)
type Client struct {
httpClient *http.Client
URL string
Logf LogfCallback
}
type LogfCallback func(format string, args ...interface{})
/*
* Discard log messages silently.
*/
func Quiet(format string, args ...interface{}) {
/* discard logs */
}
/*
* Pass log messages along to Go's "log" module.
*/
func Log(format string, args ...interface{}) {
log.Printf(format, args...)
}
/*
* Create a new Registry with the given URL and credentials, then Ping()s it
* before returning it to verify that the registry is available.
*
* You can, alternately, construct a Registry manually by populating the fields.
* This passes http.DefaultTransport to WrapTransport when creating the
* http.Client.
*/
func NewClient(registryUrl, username, password string) (*Client, error) {
transport := http.DefaultTransport
return newFromTransport(registryUrl, username, password, transport, Log)
}
/*
* Create a new Registry, as with New, using an http.Transport that disables
* SSL certificate verification.
*/
func NewInsecureClient(registryUrl, username, password string) (*Client, error) {
transport := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
return newFromTransport(registryUrl, username, password, transport, Log)
}
/*
* Given an existing http.RoundTripper such as http.DefaultTransport, build the
* transport stack necessary to authenticate to the Docker registry API. This
* adds in support for OAuth bearer tokens and HTTP Basic auth, and sets up
* error handling this library relies on.
*/
func WrapTransport(transport http.RoundTripper, url, username, password string) http.RoundTripper {
tokenTransport := &TokenTransport{
Transport: transport,
Username: username,
Password: password,
}
basicAuthTransport := &BasicTransport{
Transport: tokenTransport,
URL: url,
Username: username,
Password: password,
}
errorTransport := &ErrorTransport{
Transport: basicAuthTransport,
}
return errorTransport
}
func newFromTransport(registryUrl, username, password string, transport http.RoundTripper, logf LogfCallback) (*Client, error) {
url := strings.TrimSuffix(registryUrl, "/")
transport = WrapTransport(transport, url, username, password)
registry := &Client{
URL: url,
httpClient: &http.Client{
Transport: transport,
},
Logf: logf,
}
if err := registry.Ping(); err != nil {
return nil, err
}
return registry, nil
}
func (c *Client) url(pathTemplate string, args ...interface{}) string {
pathSuffix := fmt.Sprintf(pathTemplate, args...)
url := fmt.Sprintf("%s%s", c.URL, pathSuffix)
return url
}
func (c *Client) Ping() error {
url := c.url("/v2/")
c.Logf("registry.ping url=%s", url)
resp, err := c.httpClient.Get(url)
if err == nil {
defer resp.Body.Close()
return nil
}
return err
}

60
dist/error.go vendored
Unescape View File

@ -0,0 +1,60 @@
package dist
import (
"encoding/json"
"fmt"
"net/http"
)
type ErrorResponse struct {
HTTPStatusCode int
Errors []RegistryError `json:"errors"`
}
func (e ErrorResponse) Error() string {
errors := ""
for _, err := range e.Errors {
errors += err.Error() + "\n"
}
return errors
}
type RegistryError struct {
Code string `json:"code"`
Message string `json:"message"`
Detail json.RawMessage `json:"detail"`
}
func (r RegistryError) Error() string {
return fmt.Sprintf("%s: %s", r.Code, r.Message)
}
func (r RegistryError) Details() string {
return string(r.Detail)
}
type ErrorTransport struct {
Transport http.RoundTripper
}
func (t *ErrorTransport) RoundTrip(request *http.Request) (*http.Response, error) {
resp, err := t.Transport.RoundTrip(request)
if err != nil {
return resp, err
}
if resp.StatusCode >= 400 {
defer resp.Body.Close()
ErrResp := ErrorResponse{}
if err := json.NewDecoder(resp.Body).Decode(&ErrResp); err != nil {
return nil, fmt.Errorf("http: failed to read response body (status=%v, err=%q)", resp.StatusCode, err)
}
ErrResp.HTTPStatusCode = resp.StatusCode
return nil, &ErrResp
}
return resp, err
}

66
dist/json.go vendored
Unescape View File

@ -0,0 +1,66 @@
package dist
import (
"encoding/json"
"errors"
"net/http"
"regexp"
)
var (
ErrNoMorePages = errors.New("No more pages")
)
func (c *Client) getJson(url string, response interface{}) error {
resp, err := c.httpClient.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return err
}
return nil
}
// getPaginatedJson accepts a string and a pointer, and returns the
// next page URL while updating pointed-to variable with a parsed JSON
// value. When there are no more pages it returns `ErrNoMorePages`.
func (c *Client) getPaginatedJson(url string, response interface{}) (string, error) {
resp, err := c.httpClient.Get(url)
if err != nil {
return "", err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return "", err
}
return getNextLink(resp)
}
// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5)
// Link header. For example,
//
// <http://registry.example.com/v2/_catalog?n=5&last=tag5>; type="application/json"; rel="next"
//
// The URL is _supposed_ to be wrapped by angle brackets `< ... >`,
// but e.g., quay.io does not include them. Similarly, params like
// `rel="next"` may not have quoted values in the wild.
var nextLinkRE = regexp.MustCompile(`^ *<?([^;>]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`)
func getNextLink(resp *http.Response) (string, error) {
for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
parts := nextLinkRE.FindStringSubmatch(link)
if parts != nil {
return parts[1], nil
}
}
return "", ErrNoMorePages
}

126
dist/manifest.go vendored
Unescape View File

@ -0,0 +1,126 @@
package dist
import (
"bytes"
"io/ioutil"
"net/http"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/opencontainers/go-digest"
)
func (c *Client) Manifest(repository, reference string) (*schema1.SignedManifest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", schema1.MediaTypeManifest)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
signedManifest := &schema1.SignedManifest{}
err = signedManifest.UnmarshalJSON(body)
if err != nil {
return nil, err
}
return signedManifest, nil
}
func (c *Client) ManifestV2(repository, reference string) (*schema2.DeserializedManifest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", schema2.MediaTypeManifest)
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
deserialized := &schema2.DeserializedManifest{}
err = deserialized.UnmarshalJSON(body)
if err != nil {
return nil, err
}
return deserialized, nil
}
func (c *Client) ManifestDigest(repository, reference string) (digest.Digest, error) {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
resp, err := c.httpClient.Head(url)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return "", err
}
return digest.Parse(resp.Header.Get("Docker-Content-Digest"))
}
func (c *Client) DeleteManifest(repository string, digest digest.Digest) error {
url := c.url("/v2/%s/manifests/%s", repository, digest)
c.Logf("registry.manifest.delete url=%s repository=%s reference=%s", url, repository, digest)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
resp, err := c.httpClient.Do(req)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return err
}
return nil
}
func (c *Client) PutManifest(repository, reference string, manifest distribution.Manifest) error {
url := c.url("/v2/%s/manifests/%s", repository, reference)
c.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)
mediaType, payload, err := manifest.Payload()
if err != nil {
return err
}
buffer := bytes.NewBuffer(payload)
req, err := http.NewRequest("PUT", url, buffer)
if err != nil {
return err
}
req.Header.Set("Content-Type", mediaType)
resp, err := c.httpClient.Do(req)
if resp != nil {
defer resp.Body.Close()
}
return err
}

26
dist/repositories.go vendored
Unescape View File

@ -0,0 +1,26 @@
package dist
type repositoriesResponse struct {
Repositories []string `json:"repositories"`
}
func (c *Client) Repositories() ([]string, error) {
url := c.url("/v2/_catalog")
repos := make([]string, 0, 10)
var err error //We create this here, otherwise url will be rescoped with :=
var response repositoriesResponse
for {
c.Logf("registry.repositories url=%s", url)
url, err = c.getPaginatedJson(url, &response)
switch err {
case ErrNoMorePages:
repos = append(repos, response.Repositories...)
return repos, nil
case nil:
repos = append(repos, response.Repositories...)
continue
default:
return nil, err
}
}
}

25
dist/tags.go vendored
Unescape View File

@ -0,0 +1,25 @@
package dist
type tagsResponse struct {
Tags []string `json:"tags"`
}
func (c *Client) Tags(repository string) (tags []string, err error) {
url := c.url("/v2/%s/tags/list", repository)
var response tagsResponse
for {
c.Logf("registry.tags url=%s repository=%s", url, repository)
url, err = c.getPaginatedJson(url, &response)
switch err {
case ErrNoMorePages:
tags = append(tags, response.Tags...)
return tags, nil
case nil:
tags = append(tags, response.Tags...)
continue
default:
return nil, err
}
}
}

130
dist/tokentransport.go vendored
Unescape View File

@ -0,0 +1,130 @@
package dist
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
)
type TokenTransport struct {
Transport http.RoundTripper
Username string
Password string
}
func (t *TokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := t.Transport.RoundTrip(req)
if err != nil {
return resp, err
}
if authService := isTokenDemand(resp); authService != nil {
resp, err = t.authAndRetry(authService, req)
}
return resp, err
}
type authToken struct {
Token string `json:"token"`
}
func (t *TokenTransport) authAndRetry(authService *authService, req *http.Request) (*http.Response, error) {
token, authResp, err := t.auth(authService)
if err != nil {
return authResp, err
}
retryResp, err := t.retry(req, token)
return retryResp, err
}
func (t *TokenTransport) auth(authService *authService) (string, *http.Response, error) {
authReq, err := authService.Request(t.Username, t.Password)
if err != nil {
return "", nil, err
}
client := http.Client{
Transport: t.Transport,
}
response, err := client.Do(authReq)
if err != nil {
return "", nil, err
}
if response.StatusCode != http.StatusOK {
return "", response, err
}
defer response.Body.Close()
var authToken authToken
decoder := json.NewDecoder(response.Body)
err = decoder.Decode(&authToken)
if err != nil {
return "", nil, err
}
return authToken.Token, nil, nil
}
func (t *TokenTransport) retry(req *http.Request, token string) (*http.Response, error) {
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
resp, err := t.Transport.RoundTrip(req)
return resp, err
}
type authService struct {
Realm string
Service string
Scope string
}
func (authService *authService) Request(username, password string) (*http.Request, error) {
url, err := url.Parse(authService.Realm)
if err != nil {
return nil, err
}
q := url.Query()
q.Set("service", authService.Service)
if authService.Scope != "" {
q.Set("scope", authService.Scope)
}
url.RawQuery = q.Encode()
request, err := http.NewRequest("GET", url.String(), nil)
if err != nil {
return nil, err
}
if username != "" || password != "" {
request.SetBasicAuth(username, password)
}
return request, err
}
func isTokenDemand(resp *http.Response) *authService {
if resp == nil {
return nil
}
if resp.StatusCode != http.StatusUnauthorized {
return nil
}
return parseOauthHeader(resp)
}
func parseOauthHeader(resp *http.Response) *authService {
challenges := parseAuthHeader(resp.Header)
for _, challenge := range challenges {
if challenge.Scheme == "bearer" {
return &authService{
Realm: challenge.Parameters["realm"],
Service: challenge.Parameters["service"],
Scope: challenge.Parameters["scope"],
}
}
}
return nil
}

91
host/build.go
Unescape View File

@ -29,7 +29,8 @@ const buildHelperName = "buildhelper"
const helperPath = "/usr/lib/brand/pod/helpers/%s/" + buildHelperName
func (h *Host) RunBuild(cfg *build.ImageConfig, opts *pod.Options) error {
buildImages := make(map[string]*image.Image)
buildImages := make(map[string]*image.Repository)
for _, imgCfg := range cfg.Images {
logrus.Infof("starting image build for %s", imgCfg.Name)
if cfg.Maintainer != nil {
@ -47,20 +48,27 @@ func (h *Host) RunBuild(cfg *build.ImageConfig, opts *pod.Options) error {
for _, op := range imgCfg.CopyOperations {
logrus.Debugf("copying %s from source Image %s", op.Name, op.SourceImage)
srcRefString := reference.NewReferenceString(op.SourceImage)
srcImg, ok := buildImages[srcRefString.String()]
srcRepository, ok := buildImages[srcRefString.String()]
if !ok {
return fmt.Errorf("no image %s has bin built yet", op.SourceImage)
}
digest, ok := srcImg.Tags[srcRefString.Tag]