Browse Source

Move zone code into runner interface

master
Till Wegmüller 3 years ago
parent
commit
b20457d57f
  1. 2
      go.mod
  2. 206
      pod/container.go
  3. 2
      pod/meta/annotations.go
  4. 2
      pod/meta/manifest.go
  5. 2
      pod/meta/status.go
  6. 2
      pod/meta/status_string.go
  7. 23
      pod/runner/runner.go
  8. 228
      pod/runner/zone.go
  9. 94
      pod/zone.go

2
go.mod

@ -14,7 +14,7 @@ require (
github.com/dustin/go-humanize v1.0.0
github.com/garyburd/redigo v1.6.0 // indirect
github.com/go-test/deep v1.0.3
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/gofrs/uuid v3.2.0+incompatible
github.com/goodhosts/hostsfile v0.0.2
github.com/gorilla/handlers v1.4.2 // indirect
github.com/gotestyourself/gotestyourself v1.4.0 // indirect

206
pod/container.go

@ -9,12 +9,11 @@ import (
"path"
"time"
"git.wegmueller.it/illumos/go-zone"
"git.wegmueller.it/illumos/go-zone/config"
"git.wegmueller.it/illumos/go-zone/lifecycle"
iconf "git.wegmueller.it/opencloud/opencloud/config"
"git.wegmueller.it/opencloud/opencloud/image"
"git.wegmueller.it/opencloud/opencloud/image/reference"
"git.wegmueller.it/opencloud/opencloud/pod/meta"
"git.wegmueller.it/opencloud/opencloud/pod/runner"
"git.wegmueller.it/opencloud/opencloud/volume"
"git.wegmueller.it/opencloud/opencloud/zfs"
imageSpec "github.com/opencontainers/image-spec/specs-go/v1"
@ -25,43 +24,35 @@ import (
const (
ROOTFSDatasetName = "root"
BrandBHyve = "bhyve"
BrandKVM = "kvm"
)
type Container struct {
status Status
status meta.Status
dataset *zfs.Dataset
rootDS *zfs.Dataset
Name string `json:"name"`
UUID uuid.UUID `json:"uuid"`
Manifest *ContainerManifest `json:"manifest"`
Image image.Image `json:"image"`
ImageReference reference.Reference `json:"image_reference"`
Zone *config.Zone `json:"zone,omitempty"`
TopLayerDescriptor imageSpec.Descriptor `json:"image_layer_desc"`
Name string `json:"name"`
UUID uuid.UUID `json:"uuid"`
Manifest *meta.ContainerManifest `json:"manifest"`
Image image.Image `json:"image"`
ImageReference reference.Reference `json:"image_reference"`
TopLayerDescriptor imageSpec.Descriptor `json:"image_layer_desc"`
runner runner.Runner
volumes map[string]*volume.Volume
lifecycleManager lifecycle.Lifecyclemanager
sealed bool
}
func newContainer(manifest *ContainerManifest, name string) *Container {
c := &Container{
Name: name,
UUID: uuid.NewV4(),
Manifest: manifest,
volumes: make(map[string]*volume.Volume),
}
return c
}
func (c *Container) GetDataset() *zfs.Dataset {
return c.dataset
}
func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name string) (container *Container, rErr error) {
container = newContainer(m, name)
func CreateEmptyContainer(parentDataset *zfs.Dataset, m *meta.ContainerManifest, name string) (container *Container, rErr error) {
container = &Container{
Name: name,
UUID: uuid.NewV4(),
Manifest: m,
volumes: make(map[string]*volume.Volume),
runner: &runner.ZoneRunner{},
}
if err := container.initDataset(parentDataset); err != nil {
return nil, tracerr.Wrap(err)
@ -82,11 +73,12 @@ func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name
return nil, tracerr.Wrap(err)
}
if err := container.initNewZone(); err != nil {
return nil, tracerr.Wrap(err)
}
if err := container.register(); err != nil {
if err := container.runner.CreateContainer(runner.CreateOpts{
UUID: container.UUID,
Name: container.Name,
Path: container.Path(),
Manifest: m,
}); err != nil {
return nil, tracerr.Wrap(err)
}
@ -97,7 +89,7 @@ func CreateEmptyContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name
return container, nil
}
func CreateContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name, tag string, repo *image.Repository) (container *Container, rErr error) {
func CreateContainer(parentDataset *zfs.Dataset, m *meta.ContainerManifest, name, tag string, repo *image.Repository) (container *Container, rErr error) {
container = &Container{
UUID: uuid.NewV4(),
@ -129,15 +121,16 @@ func CreateContainer(parentDataset *zfs.Dataset, m *ContainerManifest, name, tag
return nil, tracerr.Wrap(err)
}
if err := container.initNewZone(); err != nil {
return nil, tracerr.Wrap(err)
}
container.Manifest.Spec.Root = &spec.Root{
Path: container.rootDS.Mountpoint,
}
if err := container.register(); err != nil {
if err := container.runner.CreateContainer(runner.CreateOpts{
UUID: container.UUID,
Name: container.Name,
Path: container.Path(),
Manifest: m,
}); err != nil {
return nil, tracerr.Wrap(err)
}
@ -202,57 +195,7 @@ func (c *Container) cloneImageToContainer(repo *image.Repository) error {
return nil
}
func (c *Container) initNewZone() (err error) {
c.Zone = newZoneFromContainer(c)
c.lifecycleManager, err = lifecycle.NewManager(c.Zone)
if err != nil {
return tracerr.Wrap(err)
}
if err := os.Chmod(c.dataset.Mountpoint, 0700); err != nil {
return tracerr.Wrap(err)
}
return nil
}
func (c *Container) register() (err error) {
var lockFd *os.File
if lockFd, err = config.GrabZoneLockFile(*c.Zone); err != nil {
return tracerr.Wrap(err)
}
defer config.ReleaseZoneLockFile(lockFd)
if err = c.Zone.WriteToFile(); err != nil {
return tracerr.Wrap(err)
}
if err = config.Register(c.Zone); err != nil {
return tracerr.Wrap(err)
}
manifestJSON, err := json.Marshal(c)
if err != nil {
return tracerr.Wrap(err)
}
if err := ioutil.WriteFile(c.Path("pod.json"), manifestJSON, 0440); err != nil {
return tracerr.Wrap(err)
}
if c.Zone.Brand != DefaultPodBrandName {
if err = c.lifecycleManager.Install(nil); err != nil {
return tracerr.Wrap(err)
}
}
c.sealed = true
c.status = StatusStopped
return nil
}
func (c *Container) initVolumes() error {
for i, volCfg := range c.Manifest.Volumes {
if volCfg.Name == "" {
volCfg.Name = fmt.Sprintf("volume.%d", i)
@ -313,6 +256,7 @@ func LoadContainer(parentDS *zfs.Dataset, id uuid.UUID) (*Container, error) {
if id == uuid.Nil {
panic("No UUID provided")
}
containerDS, err := parentDS.GetChildDataset(path.Join("pods", id.String()))
if err != nil {
return nil, tracerr.Wrap(err)
@ -322,6 +266,9 @@ func LoadContainer(parentDS *zfs.Dataset, id uuid.UUID) (*Container, error) {
dataset: containerDS,
UUID: id,
volumes: make(map[string]*volume.Volume),
runner: &runner.ZoneRunner{
UUID: id,
},
}
if err := container.Load(); err != nil {
@ -366,19 +313,6 @@ func (c *Container) Load() error {
return tracerr.Wrap(err)
}
if c.Zone == nil {
c.Zone = config.New(c.UUID.String())
}
if err := c.Zone.ReadFromFile(); err != nil {
return err
}
c.lifecycleManager, err = lifecycle.NewManager(c.Zone)
if err != nil {
return err
}
for _, child := range c.dataset.Children {
if child.GetName() == "root" {
continue
@ -392,6 +326,12 @@ func (c *Container) Load() error {
c.volumes[vol.GetDatasetMountPoint()] = vol
}
c.runner = &runner.ZoneRunner{
UUID: c.UUID,
Name: c.Name,
ZonePath: c.Path(),
}
c.sealed = true
return nil
}
@ -414,20 +354,11 @@ func (c *Container) Save() error {
return tracerr.Wrap(err)
}
if err = c.Zone.WriteToFile(); err != nil {
return err
}
return nil
}
func (c *Container) Status() Status {
if zone.GetZoneIdByName(c.UUID.String()) == -1 {
c.status = StatusStopped
} else {
c.status = StatusRunning
}
func (c *Container) Status() meta.Status {
c.status = c.runner.Status()
return c.status
}
@ -436,19 +367,19 @@ func (c *Container) Kill() error {
panic("container not properly loaded, hit the developer of the app who forgot to use the c.Load() function")
}
switch status := c.Status(); status {
case StatusStopped:
case meta.StatusStopped:
// All's fine
return nil
case StatusRunning:
c.status = StatusDying
case meta.StatusRunning:
c.status = meta.StatusDying
var err error
if err = c.lifecycleManager.Shutdown(nil); err != nil {
// Try a second time with halt
if err = c.lifecycleManager.Halt(); err != nil {
if err = c.runner.StopContainer(false); err != nil {
// Try a second time with halt (force)
if err = c.runner.StopContainer(true); err != nil {
return tracerr.Wrap(err)
}
}
case StatusDying:
case meta.StatusDying:
time.Sleep(250 * time.Millisecond)
return nil
default:
@ -462,35 +393,16 @@ func (c *Container) Destroy() error {
panic("conatiner not properly loaded, hit the developer of the app who forgot to use the c.Load() function")
}
if status := c.Status(); status == StatusRunning {
if status := c.Status(); status == meta.StatusRunning {
if err := c.Kill(); err != nil {
return err
}
}
if err := c.lifecycleManager.Uninstall(true, nil); err != nil {
return fmt.Errorf("can not uninstall zone %s: %w", c.UUID, err)
}
if err := c.dataset.Destroy(true); err != nil {
return fmt.Errorf("cannot destroy pod dataset for %s: %w", c.UUID, err)
}
var lockFd *os.File
var err error
if lockFd, err = config.GrabZoneLockFile(*c.Zone); err != nil {
return tracerr.Wrap(err)
}
defer config.ReleaseZoneLockFile(lockFd)
if err := config.Unregister(c.Zone); err != nil {
return fmt.Errorf("cannot unregister zone of %s: %w", c.UUID, err)
}
if err := c.Zone.RemoveFile(); err != nil {
return fmt.Errorf("can not remove zone xml of %s: %w", c.UUID, err)
}
if err := os.Remove(c.Path()); err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("error during path cleanup of %s: %w", c.UUID, err)
@ -503,9 +415,19 @@ func (c *Container) Destroy() error {
// Starts the Zone
// Only to be called after the zone setup has been done
func (c *Container) Run() error {
if c.lifecycleManager == nil || !c.sealed {
if c.runner == nil || !c.sealed {
return fmt.Errorf("container %s not properly loaded cannot start", c.Name)
}
c.status = StatusRunning
return c.lifecycleManager.Boot(nil)
if c.Status() != meta.StatusStopped {
return fmt.Errorf("can only start stopped containers")
}
if err := c.runner.StartContainer(); err != nil {
return tracerr.Wrap(err)
}
c.status = meta.StatusRunning
return nil
}

2
pod/annotations.go → pod/meta/annotations.go

@ -1,4 +1,4 @@
package pod
package meta
const (
AnnotationNamespaceBase = "org.illumos.oci"

2
pod/manifest.go → pod/meta/manifest.go

@ -1,4 +1,4 @@
package pod
package meta
import (
"fmt"

2
pod/status.go → pod/meta/status.go

@ -1,4 +1,4 @@
package pod
package meta
type Status uint

2
pod/status_string.go → pod/meta/status_string.go

@ -1,6 +1,6 @@
// Code generated by "stringer -type Status"; DO NOT EDIT.
package pod
package meta
import "strconv"

23
pod/runner/runner.go

@ -0,0 +1,23 @@
package runner
import (
"git.wegmueller.it/opencloud/opencloud/pod/meta"
uuid "github.com/satori/go.uuid"
)
type CreateOpts struct {
UUID uuid.UUID
Name string
Path string
Manifest *meta.ContainerManifest
}
type Runner interface {
CreateContainer(CreateOpts) error
StartContainer() error
StopContainer(force bool) error
DestroyContainer() error
Status() meta.Status
GetID() string
GetName() string
}

228
pod/runner/zone.go

@ -0,0 +1,228 @@
package runner
import (
"fmt"
"os"
"strconv"
"strings"
"git.wegmueller.it/illumos/go-zone"
"git.wegmueller.it/illumos/go-zone/config"
"git.wegmueller.it/illumos/go-zone/lifecycle"
"git.wegmueller.it/opencloud/opencloud/pod/meta"
uuid "github.com/satori/go.uuid"
"github.com/ztrue/tracerr"
)
const (
BrandBHyve = "bhyve"
BrandKVM = "kvm"
BrandIPKG = "ipkg"
BrandLX = "lx"
BrandPod = "pod"
BrandSparse = "sparse"
DefaultPodBrandName = BrandPod
)
type ZoneRunner struct {
UUID uuid.UUID
Name string
ZonePath string
lifecycleManager lifecycle.Lifecyclemanager
}
func (r *ZoneRunner) CreateContainer(opts CreateOpts) error {
r.UUID = opts.UUID
r.ZonePath = opts.Path
r.Name = opts.Name
zone := r.newZoneFromManifest(opts.Manifest)
var err error
r.lifecycleManager, err = lifecycle.NewManager(zone)
if err != nil {
return tracerr.Wrap(err)
}
if err := os.Chmod(r.ZonePath, 0700); err != nil {
return tracerr.Wrap(err)
}
if err := r.register(zone); err != nil {
return tracerr.Wrap(err)
}
return nil
}
func (r *ZoneRunner) StartContainer() error {
if err := r.ensureIsLoaded(); err != nil {
return tracerr.Wrap(err)
}
return r.lifecycleManager.Boot(nil)
}
func (r *ZoneRunner) StopContainer(force bool) error {
if err := r.ensureIsLoaded(); err != nil {
return tracerr.Wrap(err)
}
if force {
return r.lifecycleManager.Halt()
}
return r.lifecycleManager.Shutdown(nil)
}
func (r *ZoneRunner) DestroyContainer() error {
if err := r.ensureIsLoaded(); err != nil {
return tracerr.Wrap(err)
}
zone := r.lifecycleManager.GetZone()
if err := r.lifecycleManager.Uninstall(true, nil); err != nil {
return fmt.Errorf("can not uninstall zone %s: %w", r.UUID, err)
}
var lockFd *os.File
var err error
if lockFd, err = config.GrabZoneLockFile(zone); err != nil {
return tracerr.Wrap(err)
}
defer config.ReleaseZoneLockFile(lockFd)
if err := config.Unregister(&zone); err != nil {
return fmt.Errorf("cannot unregister zone of %s: %w", r.UUID, err)
}
if err := zone.RemoveFile(); err != nil {
return fmt.Errorf("can not remove zone xml of %s: %w", r.UUID, err)
}
return nil
}
func (r *ZoneRunner) GetID() string {
return r.UUID.String()
}
func (r *ZoneRunner) GetName() string {
return r.Name
}
func (r *ZoneRunner) Status() meta.Status {
if zone.GetZoneIdByName(r.UUID.String()) == -1 {
return meta.StatusStopped
}
return meta.StatusRunning
}
func (r ZoneRunner) ensureIsLoaded() error {
if r.lifecycleManager == nil {
var err error
if r.lifecycleManager, err = lifecycle.OpenZoneConfig(r.Name); err != nil {
return tracerr.Wrap(err)
}
}
return nil
}
func (r *ZoneRunner) register(zone *config.Zone) (err error) {
var lockFd *os.File
if lockFd, err = config.GrabZoneLockFile(*zone); err != nil {
return tracerr.Wrap(err)
}
defer config.ReleaseZoneLockFile(lockFd)
if err = zone.WriteToFile(); err != nil {
return tracerr.Wrap(err)
}
if err = config.Register(zone); err != nil {
return tracerr.Wrap(err)
}
if zone.Brand != DefaultPodBrandName {
if err = r.lifecycleManager.Install(nil); err != nil {
return tracerr.Wrap(err)
}
}
return nil
}
func (r *ZoneRunner) newZoneFromManifest(manifest *meta.ContainerManifest) *config.Zone {
z := config.New(r.UUID.String())
z.Brand = DefaultPodBrandName
z.Zonepath = r.ZonePath
z.IpType = config.IpTypeExclusive
z.State = config.ZoneStateInstalled
for anKey, anValue := range manifest.Spec.Annotations {
switch anKey {
case meta.AnnotationZoneBrand:
z.Brand = anValue
}
}
for _, mount := range manifest.Spec.Mounts {
switch mount.Type {
case "lofs":
fs := config.FileSystem{
Dir: mount.Destination,
Special: mount.Source,
Type: mount.Type,
}
if mount.Options != nil {
for _, opt := range mount.Options {
fs.Fsoption = append(fs.Fsoption, config.FSoption{Name: opt})
}
}
z.FileSystems = append(z.FileSystems, fs)
}
}
for _, vol := range manifest.Volumes {
z.Datasets = append(z.Datasets, config.Dataset{
Name: vol.Name,
})
}
if manifest.Spec.Solaris != nil {
if manifest.Spec.Solaris.LimitPriv != "" {
z.Privileges = strings.Split(manifest.Spec.Solaris.LimitPriv, ",")
}
z.SetMaxShmMemory(manifest.Spec.Solaris.MaxShmMemory)
if capCPU := manifest.Spec.Solaris.CappedCPU; capCPU != nil {
ncpu, err := strconv.ParseFloat(capCPU.Ncpus, 64)
if err != nil {
return nil
}
z.SetCappedCPU(ncpu)
}
if capMem := manifest.Spec.Solaris.CappedMemory; capMem != nil {
if capMem.Physical != "" {
z.CappedMemory = config.NewMemoryCap(capMem.Physical)
}
if capMem.Swap != "" {
z.SetMaxSwap(capMem.Swap)
}
}
for _, anet := range manifest.Spec.Solaris.Anet {
z.Networks = append(z.Networks, config.Network{
AllowedAddress: anet.Allowedaddr,
Physical: anet.Linkname,
Defrouter: anet.Defrouter,
})
}
}
return z
}

94
pod/zone.go

@ -1,94 +0,0 @@
package pod
import (
"strconv"
"strings"
"git.wegmueller.it/illumos/go-zone/config"
)
const (
DefaultPodBrandName = "pod"
)
func newZoneFromContainer(container *Container) *config.Zone {
z := config.New(container.UUID.String())
z.Brand = DefaultPodBrandName
z.Zonepath = container.Path()
z.IpType = config.IpTypeExclusive
z.State = config.ZoneStateInstalled
for anKey, anValue := range container.Image.Config.Config.Labels {
switch anKey {
case AnnotationZoneBrand:
z.Brand = anValue
}
}
for anKey, anValue := range container.Manifest.Spec.Annotations {
switch anKey {
case AnnotationZoneBrand:
z.Brand = anValue
}
}
for _, mount := range container.Manifest.Spec.Mounts {
switch mount.Type {
case "lofs":
fs := config.FileSystem{
Dir: mount.Destination,
Special: mount.Source,
Type: mount.Type,
}
if mount.Options != nil {
for _, opt := range mount.Options {
fs.Fsoption = append(fs.Fsoption, config.FSoption{Name: opt})
}
}
z.FileSystems = append(z.FileSystems, fs)
}
}
for _, vol := range container.volumes {
z.Datasets = append(z.Datasets, config.Dataset{
Name: vol.GetDatasetPath(),
})
}
if container.Manifest.Spec.Solaris != nil {
if container.Manifest.Spec.Solaris.LimitPriv != "" {
z.Privileges = strings.Split(container.Manifest.Spec.Solaris.LimitPriv, ",")
}
z.SetMaxShmMemory(container.Manifest.Spec.Solaris.MaxShmMemory)
if capCPU := container.Manifest.Spec.Solaris.CappedCPU; capCPU != nil {
ncpu, err := strconv.ParseFloat(capCPU.Ncpus, 64)
if err != nil {
return nil
}
z.SetCappedCPU(ncpu)
}
if capMem := container.Manifest.Spec.Solaris.CappedMemory; capMem != nil {
if capMem.Physical != "" {
z.CappedMemory = config.NewMemoryCap(capMem.Physical)
}
if capMem.Swap != "" {
z.SetMaxSwap(capMem.Swap)
}
}
for _, anet := range container.Manifest.Spec.Solaris.Anet {
z.Networks = append(z.Networks, config.Network{
AllowedAddress: anet.Allowedaddr,
Physical: anet.Linkname,
Defrouter: anet.Defrouter,
})
}
}
return z
}
Loading…
Cancel
Save