OpenCloud
/
opencloud
2
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity
Browse Source

CHanged: Images layers get now unpacked into a subdataset under the images dataset and not under their own image dataset

layerset
Till Wegmüller 4 years ago
parent
a4342f4382
commit
a07da8b7aa
17 changed files with 174 additions and 115 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      config/docker.go
  2. 5
      config/host.go
  3. 6
      config/image.go
  4. 9
      image/create.go
  5. 20
      image/destroy.go
  6. 11
      image/find.go
  7. 106
      image/image.go
  8. 23
      image/image_test.go
  9. 6
      image/importzone.go
  10. 9
      image/oci/layer_reader.go
  11. 19
      image/oci/writer.go
  12. 4
      image/reference/reference.go
  13. 17
      image/transport/docker.go
  14. 23
      image/unpack.go
  15. 11
      image/update.go
  16. 10
      pod/container.go
  17. 2
      zfs/dataset.go

8
config/docker.go
Unescape View File

@ -0,0 +1,8 @@
package config
const (
DockerHubURLConfigKey = "docker.hubUrl"
DockerUsernameConfigKey = "docker.username"
DockerPasswordConfigKey = "docker.password"
DockerInsecureConfigKey = "docker.insecure"
)

5
config/host.go
Unescape View File

@ -0,0 +1,5 @@
package config
const (
HostDatasetConfigKey = "root.zfs.path"
)

6
config/image.go
Unescape View File

@ -0,0 +1,6 @@
package config
const (
BlobsDirectory = "blobs"
SealSnapshotName = "seal"
)

9
image/create.go
Unescape View File

@ -1,9 +0,0 @@
package image
import (
"git.wegmueller.it/opencloud/opencloud/image/reference"
)
func (img *Image) Create(ref reference.Reference) error {
}

20
image/destroy.go
Unescape View File

@ -1,8 +1,22 @@
package image
// This Function destroys the image on disk and makes the Struct unusable
// We are relying on zfs to error when an image can not be destroyed due to still dependant images
func (img *Image) Destroy() error {
ds := img.dataset
img.dataset = nil
return ds.Destroy(true)
layerSet := make(map[string]bool)
for _, m := range img.Tags {
for i := len(m.Layers) - 1; i > 0; i-- {
lName := m.Layers[i]
if _, ok := layerSet[lName.Digest.String()]; ok {
continue
}
ds := img.GetLayerDS(lName.Digest)
//TODO Only have a layerlist that is not linked with any other image anymore
//That would mean either keep an index off all layers and in which images they are or load in all images when destroying
//The first is probably preferable
_ = ds.Destroy(true)
layerSet[lName.Digest.String()] = true
}
}
return nil
}

11
image/find.go
Unescape View File

@ -1,17 +1,14 @@
package image
import (
"path"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"git.wegmueller.it/opencloud/opencloud/zfs"
"github.com/spf13/viper"
"github.com/ztrue/tracerr"
)
func FindImage(ref *reference.Reference) *Image {
ds, err := zfs.OpenDataset(path.Join(viper.GetString("root.zfs.path"), "images", ref.GetImageName()))
func FindImage(ref reference.Reference) (*Image, error) {
ds, err := getImagesDataset()
if err != nil || ds == nil {
return nil
return nil, tracerr.Wrap(err)
}
return NewImage(ref, ds)
}

106
image/image.go
Unescape View File

@ -2,57 +2,62 @@ package image
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"git.wegmueller.it/opencloud/opencloud/zfs"
"github.com/opencontainers/go-digest"
specsv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/viper"
"github.com/ztrue/tracerr"
)
const (
BlobsDirectory = "blobs"
)
// image.Image represents the structure a predefined Image is unpacked on disk
// image.Image represents the structure of a predefined Image which unpacked on disk
// Every Image consists of layers and those layers map 1:1 to OCI Layers
// The Child Dataset rootfs contains the unpacked Root filesystem
// The Rootfs Dataset has one snapshot for every Layer of the image.
// Every Snapshot thus maps to one tarfile
// Every Container has exactly one Image
// The VFS mountpoint of the dataset contains the oci_layout of the image
// Every Layer is a child dataset of the images root dataset.
// Layers are cloned from their Parent Layer in the Chain
// Every Tag has a Manifest describing the Hierarchy of layers in it's Chain.
// Every Layer is assumed to be unique and is thus represented by the SHA256 of the tar.gz content
// Every blob is inside the blobs directory as blob the index informs which one
// In the tags.json file there will be a link between the tags and the snapshots
// Every Image has one json describing which tag has which manifest tag 1:1 -> manifest
type Image struct {
imagesDS *zfs.Dataset
Reference reference.Reference `json:"reference"`
Name string `json:"name"`
Tags map[string]specsv1.Descriptor `json:"tags"`
Reference reference.Reference `json:"reference"`
Name string `json:"name"`
Tags map[string]specsv1.Manifest `json:"tags"`
}
func NewImage(ref reference.Reference, dataset *zfs.Dataset) *Image {
// If dataset == nil assume we instantiate an empty image to create a new one
if dataset == nil {
return &Image{
Reference: ref,
Name: ref.GetImageName(),
Tags: make(map[string]specsv1.Descriptor),
}
}
return &Image{
func NewImage(ref reference.Reference, dataset *zfs.Dataset) (*Image, error) {
i := &Image{
Reference: ref,
Name: ref.GetImageName(),
Tags: make(map[string]specsv1.Descriptor),
Tags: make(map[string]specsv1.Manifest),
imagesDS: dataset,
}
if dataset == nil {
var err error
i.imagesDS, err = getImagesDataset()
if err != nil {
return nil, tracerr.Wrap(err)
}
} else {
i.imagesDS = dataset
}
return i, nil
}
func (img *Image) GetLayers() []string {
layers := make([]string, 0)
for _, child := range img.imagesDS.Children {
layers = append(layers, child.GetName())
func (img *Image) GetLayers() []digest.Digest {
tagRef, ok := img.Tags[img.Reference.Tag]
if !ok {
return nil
}
layers := make([]digest.Digest, 0)
for _, l := range tagRef.Layers {
layers = append(layers, l.Digest)
}
return layers
}
@ -66,17 +71,17 @@ func (img *Image) GetLayerDS(name digest.Digest) *zfs.Dataset {
return nil
}
func (img *Image) HasLayer(name string) bool {
func (img *Image) HasLayer(name digest.Digest) bool {
for _, child := range img.imagesDS.Children {
if child.GetName() == name {
if child.GetName() == name.Encoded() {
return true
}
}
return false
}
func (img *Image) GetManifest(digest digest.Digest) (manifest *specsv1.Manifest, err error) {
manPath := img.imagesDS.VFSPath(BlobsDirectory, digest.Algorithm().String(), digest.Encoded())
func (img *Image) GetManifestFromDisk(digest digest.Digest) (manifest *specsv1.Manifest, err error) {
manPath := img.imagesDS.VFSPath(config.BlobsDirectory, digest.Algorithm().String(), digest.Encoded())
manBlob, err := ioutil.ReadFile(manPath)
if err != nil {
return nil, tracerr.Wrap(err)
@ -88,7 +93,7 @@ func (img *Image) GetManifest(digest digest.Digest) (manifest *specsv1.Manifest,
}
func (img *Image) GetConfig(digest digest.Digest) (cfg *specsv1.Image, err error) {
cfgPath := img.imagesDS.VFSPath(BlobsDirectory, digest.Algorithm().String(), digest.Encoded())
cfgPath := img.imagesDS.VFSPath(config.BlobsDirectory, digest.Algorithm().String(), digest.Encoded())
cfgBlob, err := ioutil.ReadFile(cfgPath)
if err != nil {
return nil, tracerr.Wrap(err)
@ -99,34 +104,41 @@ func (img *Image) GetConfig(digest digest.Digest) (cfg *specsv1.Image, err error
return cfg, nil
}
func (img *Image) GetManifestDescriptor(tag string) specsv1.Descriptor {
return img.Tags[tag]
}
func LoadImage(ref reference.Reference, dataset *zfs.Dataset) (*Image, error) {
img := NewImage(ref, dataset)
if err := img.Load(); err != nil {
img, err := NewImage(ref, dataset)
if err != nil {
return nil, tracerr.Wrap(err)
}
if err = img.Load(); err != nil {
return nil, tracerr.Wrap(err)
}
return img, nil
}
func CreateImage(ref reference.Reference) (*Image, error) {
img := NewImage(ref, nil)
if err := img.Create(ref); err != nil {
func CreateImage(ref reference.Reference, dataset *zfs.Dataset) (*Image, error) {
img, err := NewImage(ref, dataset)
if err != nil {
return nil, tracerr.Wrap(err)
}
err := img.Update(ref, os.Stdout)
err = img.Update(ref, os.Stdout)
if err != nil {
return nil, tracerr.Wrap(err)
}
return img, nil
}
func CreateEmptyImage(ref reference.Reference) (*Image, error) {
img := NewImage(ref, nil)
if err := img.Create(ref); err != nil {
func CreateEmptyImage(ref reference.Reference, dataset *zfs.Dataset) (*Image, error) {
img, err := NewImage(ref, dataset)
if err != nil {
return nil, tracerr.Wrap(err)
}
return img, nil
}
func getImagesDataset() (*zfs.Dataset, error) {
rootPath := viper.GetString(config.HostDatasetConfigKey)
if rootPath == "" {
return nil, fmt.Errorf("can not get dataset: config %s is empty", config.HostDatasetConfigKey)
}
return zfs.OpenDataset(path.Join(rootPath, "images"))
}

23
image/image_test.go
Unescape View File

@ -3,6 +3,7 @@ package image
import (
"testing"
"git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"github.com/spf13/viper"
"github.com/stretchr/testify/suite"
@ -15,15 +16,15 @@ var dockerReferences = []string{
}
func init() {
viper.SetDefault("root.zfs.path", "bigtank/host")
//viper.SetDefault("docker.hubUrl", "https://registry-1.docker.io")
viper.SetDefault("docker.hubUrl", "http://localhost:5000")
viper.SetDefault("docker.username", "")
viper.SetDefault("docker.password", "")
viper.SetDefault(config.HostDatasetConfigKey, "bigtank/host")
viper.SetDefault(config.DockerHubURLConfigKey, "https://registry-1.docker.io")
viper.SetDefault(config.DockerUsernameConfigKey, "")
viper.SetDefault(config.DockerPasswordConfigKey, "")
}
type ImageTestSuite struct {
suite.Suite
images []*Image
}
func TestImageSuite(t *testing.T) {
@ -33,11 +34,19 @@ func TestImageSuite(t *testing.T) {
func (i *ImageTestSuite) TestCreate() {
for _, refStr := range dockerReferences {
ref := reference.NewReferenceString(refStr)
img, err := CreateImage(ref)
img, err := CreateImage(ref, nil)
if err != nil {
tracerr.PrintSource(err)
}
i.Require().NoError(err)
i.Require().NoError(img.Destroy())
i.images = append(i.images, img)
}
}
func (i *ImageTestSuite) AfterTest(suiteName, testName string) {
if testName == "TestCreate" {
for _, img := range i.images {
i.Require().NoError(img.Destroy())
}
}
}

6
image/importzone.go
Unescape View File

@ -31,7 +31,7 @@ func (img *Image) ImportZone(zoneName string) error {
}
// Load the RootFS contents into a OCI Image
wr, err := oci.NewOCIWriter(img.dataset.VFSPath(), digest.Canonical)
wr, err := oci.NewOCIWriter(img.imagesDS.VFSPath(), digest.Canonical)
if err != nil {
return tracerr.Wrap(err)
}
@ -48,7 +48,7 @@ func (img *Image) ImportZone(zoneName string) error {
// Load OCI Directory Layout into Empty Image (Must have been created before)
rd, err := oci.NewReader(img.dataset.VFSPath())
rd, err := oci.NewReader(img.imagesDS.VFSPath())
if err != nil {
return tracerr.Wrap(err)
}
@ -58,5 +58,5 @@ func (img *Image) ImportZone(zoneName string) error {
return tracerr.Wrap(err)
}
return img.UnpackOCI(img.dataset.VFSPath(), manifest)
return img.UnpackOCI(img.imagesDS.VFSPath(), manifest)
}

9
image/oci/layer_reader.go
Unescape View File

@ -9,13 +9,12 @@ import (
"path/filepath"
"strings"
"git.wegmueller.it/opencloud/opencloud/config"
"github.com/opencontainers/go-digest"
specsv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/ztrue/tracerr"
)
const BlobsDirectory = "blobs"
type LayerReader struct {
digest digest.Digest
srcPath string
@ -28,7 +27,7 @@ func NewLayerReader(ociDir string, layer specsv1.Descriptor) (*LayerReader, erro
l := &LayerReader{
digest: layer.Digest,
}
l.srcPath = filepath.Join(ociDir, BlobsDirectory, l.digest.Algorithm().String(), l.digest.Encoded())
l.srcPath = filepath.Join(ociDir, config.BlobsDirectory, l.digest.Algorithm().String(), l.digest.Encoded())
f, err := os.Open(l.srcPath)
if err != nil {
return nil, tracerr.Wrap(err)
@ -62,7 +61,7 @@ func (l *LayerReader) ExtractTreeInto(dir string) error {
}
if strings.Contains(th.Name, ".wh.") {
path := filepath.Join(dir, strings.Replace(th.Name, ".wh.", "", -1))
if err := os.Remove(path); err != nil {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return tracerr.Wrap(err)
}
_, _ = io.Copy(ioutil.Discard, l.archiveReader)
@ -104,7 +103,7 @@ func (l *LayerReader) Close() error {
func unpackDir(th *tar.Header, path string) error {
info := th.FileInfo()
if err := os.Mkdir(path, info.Mode()); err != nil {
if err := os.Mkdir(path, info.Mode()); err != nil && !os.IsExist(err) {
return tracerr.Wrap(err)
}
if err := os.Chown(path, th.Uid, th.Gid); err != nil {

19
image/oci/writer.go
Unescape View File

@ -1,7 +1,6 @@
package oci
import (
"bytes"
"encoding/hex"
"encoding/json"
"io"
@ -125,24 +124,30 @@ func (w *Writer) GetBlobPath(encoded string) string {
}
func (w *Writer) AddConfig(config *specsv1.Image) error {
buff := bytes.NewBuffer(nil)
tmpFile, err := ioutil.TempFile(w.GetBlobPath(""), ConfigTempFilePattern)
if err != nil {
return tracerr.Wrap(err)
}
h := w.algorithm.Hash()
if err := json.NewEncoder(io.MultiWriter(h, buff)).Encode(config); err != nil {
if err := json.NewEncoder(io.MultiWriter(h, tmpFile)).Encode(config); err != nil {
return tracerr.Wrap(err)
}
stat, err := tmpFile.Stat()
if err != nil {
return tracerr.Wrap(err)
}
cfgDesc := specsv1.Descriptor{
w.cfg = specsv1.Descriptor{
MediaType: specsv1.MediaTypeImageConfig,
Digest: digest.NewDigest(w.algorithm, h),
Size: int64(buff.Cap()),
Size: stat.Size(),
Platform: &specsv1.Platform{
OS: runtime.GOOS,
Architecture: runtime.GOARCH,
},
}
if err := ioutil.WriteFile(w.GetBlobPath(cfgDesc.Digest.Encoded()), buff.Bytes(), 0644); err != nil {
if err := os.Rename(tmpFile.Name(), w.GetBlobPath(w.cfg.Digest.Encoded())); err != nil {
return tracerr.Wrap(err)
}
w.cfg = cfgDesc
return nil
}

4
image/reference/reference.go
Unescape View File

@ -42,8 +42,8 @@ func (r *Reference) IsLocal() bool {
return false
}
func NewReferenceString(refString string) *Reference {
r := &Reference{}
func NewReferenceString(refString string) Reference {
r := Reference{}
r.Parse(refString)
return r
}

17
image/transport/docker.go
Unescape View File

@ -10,6 +10,7 @@ import (
"path/filepath"
"sync"
"git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"github.com/docker/distribution"
"github.com/heroku/docker-registry-client/registry"
@ -22,11 +23,11 @@ import (
"github.com/ztrue/tracerr"
)
func PullDockerImage(ref reference.Reference, target string, progessOutput *os.File, skipList []string) (*specsv1.Descriptor, error) {
registryURL := viper.GetString("docker.hubUrl")
username := viper.GetString("docker.username")
password := viper.GetString("docker.password")
insecure := viper.GetBool("docker.insecure")
func PullDockerImage(ref reference.Reference, target string, progessOutput *os.File, skipList []digest.Digest) (*specsv1.Descriptor, error) {
registryURL := viper.GetString(config.DockerHubURLConfigKey)
username := viper.GetString(config.DockerUsernameConfigKey)
password := viper.GetString(config.DockerPasswordConfigKey)
insecure := viper.GetBool(config.DockerInsecureConfigKey)
if ref.Host != "" {
if insecure {
@ -64,7 +65,7 @@ func PullDockerImage(ref reference.Reference, target string, progessOutput *os.F
return nil, tracerr.Wrap(err)
}
}
if skipListContains(skipList, l.Digest.Encoded()) {
if skipListContains(skipList, l.Digest) {
continue
}
tgtPath := filepath.Join(target, l.Digest.Algorithm().String(), l.Digest.Encoded())
@ -136,9 +137,9 @@ func PullDockerImage(ref reference.Reference, target string, progessOutput *os.F
return &manDesc, nil
}
func skipListContains(skipList []string, encoded string) bool {
func skipListContains(skipList []digest.Digest, encoded digest.Digest) bool {
for _, e := range skipList {
if e == encoded {
if e.String() == encoded.String() {
return true
}
}

23
image/unpack.go
Unescape View File

@ -2,7 +2,9 @@ package image
import (
"os"
"path"
"git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image/oci"
"git.wegmueller.it/opencloud/opencloud/zfs"
specsv1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -23,20 +25,31 @@ func (img *Image) UnpackOCI(ociDir string, manifest *specsv1.Manifest) (Gerr err
}()
for i, layer := range manifest.Layers {
if img.HasLayer(layer.Digest.Encoded()) {
if img.HasLayer(layer.Digest) {
continue
}
var layerDS *zfs.Dataset
if parentDS := img.GetLayerDS(manifest.Layers[i-1].Digest); parentDS != nil && i != 0 {
snap, err := parentDS.GetSnapshot(SealSnapshotName)
if i == 0 {
var err error
layerDS, err = img.imagesDS.CreateChildDataset(layer.Digest.Encoded(), zfs.Properties{
zfs.PropertyCompression: zfs.CompressionLZ4,
})
if err != nil {
return tracerr.Wrap(err)
}
} else if parentDS := img.GetLayerDS(manifest.Layers[i-1].Digest); parentDS != nil {
snap, err := parentDS.GetSnapshot(config.SealSnapshotName)
if err != nil {
return tracerr.Wrap(err)
}
layerDS, err = snap.Clone(layer.Digest.Encoded(), zfs.Properties{
layerDS, err = snap.Clone(path.Join(img.imagesDS.Path, layer.Digest.Encoded()), zfs.Properties{
zfs.PropertyCompression: zfs.CompressionLZ4,
})
if err != nil {
return tracerr.Wrap(err)
}
} else {
var err error
layerDS, err = img.imagesDS.CreateChildDataset(layer.Digest.Encoded(), zfs.Properties{
@ -57,7 +70,7 @@ func (img *Image) UnpackOCI(ociDir string, manifest *specsv1.Manifest) (Gerr err
return tracerr.Wrap(err)
}
_, err = layerDS.Snapshot(SealSnapshotName)
_, err = layerDS.Snapshot(config.SealSnapshotName)
if err != nil {
return tracerr.Wrap(err)
}

11
image/update.go
Unescape View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"git.wegmueller.it/opencloud/opencloud/image/transport"
specsv1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -20,7 +21,7 @@ func (img *Image) Update(ref reference.Reference, progessOutput *os.File) (err e
}
// Check URL and Download the reference
targetDir := img.imagesDS.VFSPath(BlobsDirectory)
targetDir := img.imagesDS.VFSPath(config.BlobsDirectory)
if _, err := os.Stat(targetDir); os.IsNotExist(err) {
err = os.MkdirAll(targetDir, 0755)
@ -38,7 +39,7 @@ func (img *Image) Update(ref reference.Reference, progessOutput *os.File) (err e
return tracerr.Wrap(err)
}
manifest, err := img.GetManifest(manDesc.Digest)
manifest, err := img.GetManifestFromDisk(manDesc.Digest)
if err != nil {
return tracerr.Wrap(err)
}
@ -47,8 +48,7 @@ func (img *Image) Update(ref reference.Reference, progessOutput *os.File) (err e
return tracerr.Wrap(err)
}
img.meta.AddManifest(*manDesc)
img.meta.Tags[ref.Tag] = *manDesc
img.Tags[ref.Tag] = *manifest
}
if ref.Scheme == reference.SchemeACI {
@ -58,10 +58,9 @@ func (img *Image) Update(ref reference.Reference, progessOutput *os.File) (err e
//TODO Unpack ACI
}
if err := img.meta.Save(img.dataset.VFSPath()); err != nil {
if err := img.Save(); err != nil {
return tracerr.Wrap(err)
}
img.sealed = true
return nil
}

10
pod/container.go
Unescape View File

@ -14,6 +14,7 @@ import (
"git.wegmueller.it/illumos/go-zone/config"
"git.wegmueller.it/illumos/go-zone/lifecycle"
"git.wegmueller.it/opencloud/opencloud/common"
iconf "git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"git.wegmueller.it/opencloud/opencloud/zfs"
@ -53,10 +54,9 @@ func newContainer(parentDS *zfs.Dataset, id uuid.UUID) *Container {
}
func CreateContainer(parentDataset *zfs.Dataset, pm spec.Spec, tag string, img *image.Image) (container *Container, rErr error) {
manDesc := img.GetManifestDescriptor(tag)
manifest, err := img.GetManifest(manDesc.Digest)
if err != nil {
return nil, tracerr.Wrap(err)
manifest, ok := img.Tags[tag]
if !ok {
return nil, fmt.Errorf("no manifest for tag %s in image %s", tag, img.Name)
}
cfg, err := img.GetConfig(manifest.Config.Digest)
if err != nil {
@ -95,7 +95,7 @@ func CreateContainer(parentDataset *zfs.Dataset, pm spec.Spec, tag string, img *
return nil, fmt.Errorf("could not find dataset for layer hash %s", topLayer.Digest.String())
}
layerSnap, err := layerDS.GetSnapshot(image.SealSnapshotName)
layerSnap, err := layerDS.GetSnapshot(iconf.SealSnapshotName)
if err != nil {
return nil, tracerr.Wrap(err)
}

2
zfs/dataset.go
Unescape View File

@ -261,7 +261,7 @@ func (d *Dataset) GetSnapshot(name string) (*Dataset, error) {
if err != nil {
return ds, nil
}
return &Dataset{}, err
return ds, err
}
func (d *Dataset) Destroy(recursive bool) error {

Write Preview
Loading…
Cancel
Save