diff --git a/go.mod b/go.mod index 9872228..129ead1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/garyburd/redigo v1.6.0 // indirect github.com/go-test/deep v1.0.3 - github.com/gofrs/uuid v3.2.0+incompatible // indirect + github.com/gofrs/uuid v3.2.0+incompatible github.com/goodhosts/hostsfile v0.0.2 github.com/gorilla/handlers v1.4.2 // indirect github.com/gotestyourself/gotestyourself v1.4.0 // indirect diff --git a/pod/container.go b/pod/container.go index e4caf1a..66c5afc 100644 --- a/pod/container.go +++ b/pod/container.go @@ -9,12 +9,11 @@ import ( "path" "time" - "git.wegmueller.it/illumos/go-zone" - "git.wegmueller.it/illumos/go-zone/config" - "git.wegmueller.it/illumos/go-zone/lifecycle" iconf "git.wegmueller.it/opencloud/opencloud/config" "git.wegmueller.it/opencloud/opencloud/image" "git.wegmueller.it/opencloud/opencloud/image/reference" + "git.wegmueller.it/opencloud/opencloud/pod/meta" + "git.wegmueller.it/opencloud/opencloud/pod/runner" "git.wegmueller.it/opencloud/opencloud/volume" "git.wegmueller.it/opencloud/opencloud/zfs" imageSpec "github.com/opencontainers/image-spec/specs-go/v1" @@ -25,43 +24,35 @@ import ( const ( ROOTFSDatasetName = "root" - BrandBHyve = "bhyve" - BrandKVM = "kvm" ) type Container struct { - status Status + status meta.Status dataset *zfs.Dataset rootDS *zfs.Dataset - Name string `json:"name"` - UUID uuid.UUID `json:"uuid"` - Manifest *ContainerManifest `json:"manifest"` - Image image.Image `json:"image"` - ImageReference reference.Reference `json:"image_reference"` - Zone *config.Zone `json:"zone,omitempty"` - TopLayerDescriptor imageSpec.Descriptor `json:"image_layer_desc"` + Name string `json:"name"` + UUID uuid.UUID `json:"uuid"` + Manifest *meta.ContainerManifest `json:"manifest"` + Image image.Image `json:"image"` + ImageReference reference.Reference `json:"image_reference"` + TopLayerDescriptor imageSpec.Descriptor `json:"image_layer_desc"` + runner runner.Runner volumes map[string]*volume.Volume - lifecycleManager lifecycle.Lifecyclemanager sealed bool } -func newContainer(manifest *ContainerManifest, name string) *Container { - c := &Container{ - Name: name, - UUID: uuid.NewV4(), - Manifest: manifest, - volumes: make(map[string]*volume.Volume), - } - - return c -} - func (c *Container) GetDataset() *zfs.Dataset { return c.dataset } -func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name string) (container *Container, rErr error) { - container = newContainer(m, name) +func CreateEmptyContainer(parentDataset *zfs.Dataset, m *meta.ContainerManifest, name string) (container *Container, rErr error) { + container = &Container{ + Name: name, + UUID: uuid.NewV4(), + Manifest: m, + volumes: make(map[string]*volume.Volume), + runner: &runner.ZoneRunner{}, + } if err := container.initDataset(parentDataset); err != nil { return nil, tracerr.Wrap(err) @@ -82,11 +73,12 @@ func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name return nil, tracerr.Wrap(err) } - if err := container.initNewZone(); err != nil { - return nil, tracerr.Wrap(err) - } - - if err := container.register(); err != nil { + if err := container.runner.CreateContainer(runner.CreateOpts{ + UUID: container.UUID, + Name: container.Name, + Path: container.Path(), + Manifest: m, + }); err != nil { return nil, tracerr.Wrap(err) } @@ -97,7 +89,7 @@ func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name return container, nil } -func CreateContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name, tag string, repo *image.Repository) (container *Container, rErr error) { +func CreateContainer(parentDataset *zfs.Dataset, m *meta.ContainerManifest, name, tag string, repo *image.Repository) (container *Container, rErr error) { container = &Container{ UUID: uuid.NewV4(), @@ -129,15 +121,16 @@ func CreateContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name, tag return nil, tracerr.Wrap(err) } - if err := container.initNewZone(); err != nil { - return nil, tracerr.Wrap(err) - } - container.Manifest.Spec.Root = &spec.Root{ Path: container.rootDS.Mountpoint, } - if err := container.register(); err != nil { + if err := container.runner.CreateContainer(runner.CreateOpts{ + UUID: container.UUID, + Name: container.Name, + Path: container.Path(), + Manifest: m, + }); err != nil { return nil, tracerr.Wrap(err) } @@ -202,57 +195,7 @@ func (c *Container) cloneImageToContainer(repo *image.Repository) error { return nil } -func (c *Container) initNewZone() (err error) { - c.Zone = newZoneFromContainer(c) - - c.lifecycleManager, err = lifecycle.NewManager(c.Zone) - if err != nil { - return tracerr.Wrap(err) - } - - if err := os.Chmod(c.dataset.Mountpoint, 0700); err != nil { - return tracerr.Wrap(err) - } - - return nil -} - -func (c *Container) register() (err error) { - var lockFd *os.File - if lockFd, err = config.GrabZoneLockFile(*c.Zone); err != nil { - return tracerr.Wrap(err) - } - defer config.ReleaseZoneLockFile(lockFd) - if err = c.Zone.WriteToFile(); err != nil { - return tracerr.Wrap(err) - } - - if err = config.Register(c.Zone); err != nil { - return tracerr.Wrap(err) - } - manifestJSON, err := json.Marshal(c) - if err != nil { - return tracerr.Wrap(err) - } - - if err := ioutil.WriteFile(c.Path("pod.json"), manifestJSON, 0440); err != nil { - return tracerr.Wrap(err) - } - - if c.Zone.Brand != DefaultPodBrandName { - if err = c.lifecycleManager.Install(nil); err != nil { - return tracerr.Wrap(err) - } - } - - c.sealed = true - c.status = StatusStopped - - return nil -} - func (c *Container) initVolumes() error { - for i, volCfg := range c.Manifest.Volumes { if volCfg.Name == "" { volCfg.Name = fmt.Sprintf("volume.%d", i) @@ -313,6 +256,7 @@ func LoadContainer(parentDS *zfs.Dataset, id uuid.UUID) (*Container, error) { if id == uuid.Nil { panic("No UUID provided") } + containerDS, err := parentDS.GetChildDataset(path.Join("pods", id.String())) if err != nil { return nil, tracerr.Wrap(err) @@ -322,6 +266,9 @@ func LoadContainer(parentDS *zfs.Dataset, id uuid.UUID) (*Container, error) { dataset: containerDS, UUID: id, volumes: make(map[string]*volume.Volume), + runner: &runner.ZoneRunner{ + UUID: id, + }, } if err := container.Load(); err != nil { @@ -366,19 +313,6 @@ func (c *Container) Load() error { return tracerr.Wrap(err) } - if c.Zone == nil { - c.Zone = config.New(c.UUID.String()) - } - - if err := c.Zone.ReadFromFile(); err != nil { - return err - } - - c.lifecycleManager, err = lifecycle.NewManager(c.Zone) - if err != nil { - return err - } - for _, child := range c.dataset.Children { if child.GetName() == "root" { continue @@ -392,6 +326,12 @@ func (c *Container) Load() error { c.volumes[vol.GetDatasetMountPoint()] = vol } + c.runner = &runner.ZoneRunner{ + UUID: c.UUID, + Name: c.Name, + ZonePath: c.Path(), + } + c.sealed = true return nil } @@ -414,20 +354,11 @@ func (c *Container) Save() error { return tracerr.Wrap(err) } - if err = c.Zone.WriteToFile(); err != nil { - return err - } - return nil } -func (c *Container) Status() Status { - if zone.GetZoneIdByName(c.UUID.String()) == -1 { - c.status = StatusStopped - } else { - c.status = StatusRunning - } - +func (c *Container) Status() meta.Status { + c.status = c.runner.Status() return c.status } @@ -436,19 +367,19 @@ func (c *Container) Kill() error { panic("container not properly loaded, hit the developer of the app who forgot to use the c.Load() function") } switch status := c.Status(); status { - case StatusStopped: + case meta.StatusStopped: // All's fine return nil - case StatusRunning: - c.status = StatusDying + case meta.StatusRunning: + c.status = meta.StatusDying var err error - if err = c.lifecycleManager.Shutdown(nil); err != nil { - // Try a second time with halt - if err = c.lifecycleManager.Halt(); err != nil { + if err = c.runner.StopContainer(false); err != nil { + // Try a second time with halt (force) + if err = c.runner.StopContainer(true); err != nil { return tracerr.Wrap(err) } } - case StatusDying: + case meta.StatusDying: time.Sleep(250 * time.Millisecond) return nil default: @@ -462,35 +393,16 @@ func (c *Container) Destroy() error { panic("conatiner not properly loaded, hit the developer of the app who forgot to use the c.Load() function") } - if status := c.Status(); status == StatusRunning { + if status := c.Status(); status == meta.StatusRunning { if err := c.Kill(); err != nil { return err } } - if err := c.lifecycleManager.Uninstall(true, nil); err != nil { - return fmt.Errorf("can not uninstall zone %s: %w", c.UUID, err) - } - if err := c.dataset.Destroy(true); err != nil { return fmt.Errorf("cannot destroy pod dataset for %s: %w", c.UUID, err) } - var lockFd *os.File - var err error - if lockFd, err = config.GrabZoneLockFile(*c.Zone); err != nil { - return tracerr.Wrap(err) - } - defer config.ReleaseZoneLockFile(lockFd) - - if err := config.Unregister(c.Zone); err != nil { - return fmt.Errorf("cannot unregister zone of %s: %w", c.UUID, err) - } - - if err := c.Zone.RemoveFile(); err != nil { - return fmt.Errorf("can not remove zone xml of %s: %w", c.UUID, err) - } - if err := os.Remove(c.Path()); err != nil { if !os.IsNotExist(err) { return fmt.Errorf("error during path cleanup of %s: %w", c.UUID, err) @@ -503,9 +415,19 @@ func (c *Container) Destroy() error { // Starts the Zone // Only to be called after the zone setup has been done func (c *Container) Run() error { - if c.lifecycleManager == nil || !c.sealed { + if c.runner == nil || !c.sealed { return fmt.Errorf("container %s not properly loaded cannot start", c.Name) } - c.status = StatusRunning - return c.lifecycleManager.Boot(nil) + + if c.Status() != meta.StatusStopped { + return fmt.Errorf("can only start stopped containers") + } + + if err := c.runner.StartContainer(); err != nil { + return tracerr.Wrap(err) + } + + c.status = meta.StatusRunning + + return nil } diff --git a/pod/annotations.go b/pod/meta/annotations.go similarity index 94% rename from pod/annotations.go rename to pod/meta/annotations.go index be29fb0..7dca2a5 100644 --- a/pod/annotations.go +++ b/pod/meta/annotations.go @@ -1,4 +1,4 @@ -package pod +package meta const ( AnnotationNamespaceBase = "org.illumos.oci" diff --git a/pod/manifest.go b/pod/meta/manifest.go similarity index 99% rename from pod/manifest.go rename to pod/meta/manifest.go index 66f59e8..f42fef1 100644 --- a/pod/manifest.go +++ b/pod/meta/manifest.go @@ -1,4 +1,4 @@ -package pod +package meta import ( "fmt" diff --git a/pod/status.go b/pod/meta/status.go similarity index 89% rename from pod/status.go rename to pod/meta/status.go index 48b6263..04421d5 100644 --- a/pod/status.go +++ b/pod/meta/status.go @@ -1,4 +1,4 @@ -package pod +package meta type Status uint diff --git a/pod/status_string.go b/pod/meta/status_string.go similarity index 96% rename from pod/status_string.go rename to pod/meta/status_string.go index 23d80f8..5ed1833 100644 --- a/pod/status_string.go +++ b/pod/meta/status_string.go @@ -1,6 +1,6 @@ // Code generated by "stringer -type Status"; DO NOT EDIT. -package pod +package meta import "strconv" diff --git a/pod/runner/runner.go b/pod/runner/runner.go new file mode 100644 index 0000000..0da84b7 --- /dev/null +++ b/pod/runner/runner.go @@ -0,0 +1,23 @@ +package runner + +import ( + "git.wegmueller.it/opencloud/opencloud/pod/meta" + uuid "github.com/satori/go.uuid" +) + +type CreateOpts struct { + UUID uuid.UUID + Name string + Path string + Manifest *meta.ContainerManifest +} + +type Runner interface { + CreateContainer(CreateOpts) error + StartContainer() error + StopContainer(force bool) error + DestroyContainer() error + Status() meta.Status + GetID() string + GetName() string +} diff --git a/pod/runner/zone.go b/pod/runner/zone.go new file mode 100644 index 0000000..f6a30c4 --- /dev/null +++ b/pod/runner/zone.go @@ -0,0 +1,228 @@ +package runner + +import ( + "fmt" + "os" + "strconv" + "strings" + + "git.wegmueller.it/illumos/go-zone" + "git.wegmueller.it/illumos/go-zone/config" + "git.wegmueller.it/illumos/go-zone/lifecycle" + "git.wegmueller.it/opencloud/opencloud/pod/meta" + uuid "github.com/satori/go.uuid" + "github.com/ztrue/tracerr" +) + +const ( + BrandBHyve = "bhyve" + BrandKVM = "kvm" + BrandIPKG = "ipkg" + BrandLX = "lx" + BrandPod = "pod" + BrandSparse = "sparse" + DefaultPodBrandName = BrandPod +) + +type ZoneRunner struct { + UUID uuid.UUID + Name string + ZonePath string + lifecycleManager lifecycle.Lifecyclemanager +} + +func (r *ZoneRunner) CreateContainer(opts CreateOpts) error { + r.UUID = opts.UUID + r.ZonePath = opts.Path + r.Name = opts.Name + + zone := r.newZoneFromManifest(opts.Manifest) + + var err error + r.lifecycleManager, err = lifecycle.NewManager(zone) + if err != nil { + return tracerr.Wrap(err) + } + + if err := os.Chmod(r.ZonePath, 0700); err != nil { + return tracerr.Wrap(err) + } + + if err := r.register(zone); err != nil { + return tracerr.Wrap(err) + } + + return nil +} + +func (r *ZoneRunner) StartContainer() error { + if err := r.ensureIsLoaded(); err != nil { + return tracerr.Wrap(err) + } + + return r.lifecycleManager.Boot(nil) +} + +func (r *ZoneRunner) StopContainer(force bool) error { + if err := r.ensureIsLoaded(); err != nil { + return tracerr.Wrap(err) + } + + if force { + return r.lifecycleManager.Halt() + } + return r.lifecycleManager.Shutdown(nil) +} + +func (r *ZoneRunner) DestroyContainer() error { + if err := r.ensureIsLoaded(); err != nil { + return tracerr.Wrap(err) + } + + zone := r.lifecycleManager.GetZone() + + if err := r.lifecycleManager.Uninstall(true, nil); err != nil { + return fmt.Errorf("can not uninstall zone %s: %w", r.UUID, err) + } + + var lockFd *os.File + var err error + if lockFd, err = config.GrabZoneLockFile(zone); err != nil { + return tracerr.Wrap(err) + } + defer config.ReleaseZoneLockFile(lockFd) + + if err := config.Unregister(&zone); err != nil { + return fmt.Errorf("cannot unregister zone of %s: %w", r.UUID, err) + } + + if err := zone.RemoveFile(); err != nil { + return fmt.Errorf("can not remove zone xml of %s: %w", r.UUID, err) + } + + return nil +} + +func (r *ZoneRunner) GetID() string { + return r.UUID.String() +} +func (r *ZoneRunner) GetName() string { + return r.Name +} + +func (r *ZoneRunner) Status() meta.Status { + if zone.GetZoneIdByName(r.UUID.String()) == -1 { + return meta.StatusStopped + } + return meta.StatusRunning +} + +func (r ZoneRunner) ensureIsLoaded() error { + if r.lifecycleManager == nil { + var err error + if r.lifecycleManager, err = lifecycle.OpenZoneConfig(r.Name); err != nil { + return tracerr.Wrap(err) + } + } + + return nil +} + +func (r *ZoneRunner) register(zone *config.Zone) (err error) { + var lockFd *os.File + if lockFd, err = config.GrabZoneLockFile(*zone); err != nil { + return tracerr.Wrap(err) + } + defer config.ReleaseZoneLockFile(lockFd) + if err = zone.WriteToFile(); err != nil { + return tracerr.Wrap(err) + } + + if err = config.Register(zone); err != nil { + return tracerr.Wrap(err) + } + + if zone.Brand != DefaultPodBrandName { + if err = r.lifecycleManager.Install(nil); err != nil { + return tracerr.Wrap(err) + } + } + + return nil +} + +func (r *ZoneRunner) newZoneFromManifest(manifest *meta.ContainerManifest) *config.Zone { + + z := config.New(r.UUID.String()) + z.Brand = DefaultPodBrandName + z.Zonepath = r.ZonePath + z.IpType = config.IpTypeExclusive + z.State = config.ZoneStateInstalled + + for anKey, anValue := range manifest.Spec.Annotations { + switch anKey { + case meta.AnnotationZoneBrand: + z.Brand = anValue + } + } + + for _, mount := range manifest.Spec.Mounts { + switch mount.Type { + case "lofs": + fs := config.FileSystem{ + Dir: mount.Destination, + Special: mount.Source, + Type: mount.Type, + } + if mount.Options != nil { + for _, opt := range mount.Options { + fs.Fsoption = append(fs.Fsoption, config.FSoption{Name: opt}) + } + } + z.FileSystems = append(z.FileSystems, fs) + } + } + + for _, vol := range manifest.Volumes { + z.Datasets = append(z.Datasets, config.Dataset{ + Name: vol.Name, + }) + } + + if manifest.Spec.Solaris != nil { + if manifest.Spec.Solaris.LimitPriv != "" { + z.Privileges = strings.Split(manifest.Spec.Solaris.LimitPriv, ",") + } + + z.SetMaxShmMemory(manifest.Spec.Solaris.MaxShmMemory) + + if capCPU := manifest.Spec.Solaris.CappedCPU; capCPU != nil { + ncpu, err := strconv.ParseFloat(capCPU.Ncpus, 64) + if err != nil { + return nil + } + z.SetCappedCPU(ncpu) + } + + if capMem := manifest.Spec.Solaris.CappedMemory; capMem != nil { + if capMem.Physical != "" { + z.CappedMemory = config.NewMemoryCap(capMem.Physical) + } + if capMem.Swap != "" { + z.SetMaxSwap(capMem.Swap) + } + } + + for _, anet := range manifest.Spec.Solaris.Anet { + z.Networks = append(z.Networks, config.Network{ + AllowedAddress: anet.Allowedaddr, + Physical: anet.Linkname, + Defrouter: anet.Defrouter, + }) + } + + } + + return z + +} diff --git a/pod/zone.go b/pod/zone.go deleted file mode 100644 index 16f4d2f..0000000 --- a/pod/zone.go +++ /dev/null @@ -1,94 +0,0 @@ -package pod - -import ( - "strconv" - "strings" - - "git.wegmueller.it/illumos/go-zone/config" -) - -const ( - DefaultPodBrandName = "pod" -) - -func newZoneFromContainer(container *Container) *config.Zone { - z := config.New(container.UUID.String()) - z.Brand = DefaultPodBrandName - z.Zonepath = container.Path() - z.IpType = config.IpTypeExclusive - z.State = config.ZoneStateInstalled - - for anKey, anValue := range container.Image.Config.Config.Labels { - switch anKey { - case AnnotationZoneBrand: - z.Brand = anValue - } - } - - for anKey, anValue := range container.Manifest.Spec.Annotations { - switch anKey { - case AnnotationZoneBrand: - z.Brand = anValue - } - } - - for _, mount := range container.Manifest.Spec.Mounts { - switch mount.Type { - case "lofs": - fs := config.FileSystem{ - Dir: mount.Destination, - Special: mount.Source, - Type: mount.Type, - } - if mount.Options != nil { - for _, opt := range mount.Options { - fs.Fsoption = append(fs.Fsoption, config.FSoption{Name: opt}) - } - } - z.FileSystems = append(z.FileSystems, fs) - } - } - - for _, vol := range container.volumes { - z.Datasets = append(z.Datasets, config.Dataset{ - Name: vol.GetDatasetPath(), - }) - } - - if container.Manifest.Spec.Solaris != nil { - if container.Manifest.Spec.Solaris.LimitPriv != "" { - z.Privileges = strings.Split(container.Manifest.Spec.Solaris.LimitPriv, ",") - } - - z.SetMaxShmMemory(container.Manifest.Spec.Solaris.MaxShmMemory) - - if capCPU := container.Manifest.Spec.Solaris.CappedCPU; capCPU != nil { - ncpu, err := strconv.ParseFloat(capCPU.Ncpus, 64) - if err != nil { - return nil - } - z.SetCappedCPU(ncpu) - } - - if capMem := container.Manifest.Spec.Solaris.CappedMemory; capMem != nil { - if capMem.Physical != "" { - z.CappedMemory = config.NewMemoryCap(capMem.Physical) - } - if capMem.Swap != "" { - z.SetMaxSwap(capMem.Swap) - } - } - - for _, anet := range container.Manifest.Spec.Solaris.Anet { - z.Networks = append(z.Networks, config.Network{ - AllowedAddress: anet.Allowedaddr, - Physical: anet.Linkname, - Defrouter: anet.Defrouter, - }) - } - - } - - return z - -}