add global resource limmits

engine/compiler/compiler.go

@@ -41,13 +41,12 @@ var Privileged = []string{
 // Resources defines container resource constraints. These
 // constraints are per-container, not per-pipeline.
 type Resources struct {
-	MemLimit     int64
-	MemSwapLimit int64
-	ShmSize      int64
-	CPUQuota     int64
-	CPUPeriod    int64
-	CPUShares    int64
-	CPUSet       []string
+	Memory     int64
+	MemorySwap int64
+	CPUQuota   int64
+	CPUPeriod  int64
+	CPUShares  int64
+	CPUSet     []string
 }
 
 // Args provides compiler arguments.
@@ -373,9 +372,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
 
 	// append global resource limits to steps
 	for _, step := range spec.Steps {
-		step.MemSwapLimit = c.Resources.MemSwapLimit
-		step.MemLimit = c.Resources.MemLimit
-		step.ShmSize = c.Resources.ShmSize
+		step.MemSwapLimit = c.Resources.MemorySwap
+		step.MemLimit = c.Resources.Memory
 		step.CPUPeriod = c.Resources.CPUPeriod
 		step.CPUQuota = c.Resources.CPUQuota
 		step.CPUShares = c.Resources.CPUShares

engine/compiler/step.go

@@ -42,7 +42,6 @@ func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step {
 		//
 
 		Networks: nil, // set in compiler.go
-		Files:    nil, // set below
 		Volumes:  nil, // set below
 		Devices:  nil, // see below
 		// Resources:    toResources(src), // TODO

engine/convert.go

@@ -73,19 +73,16 @@ func toHostConfig(spec *Spec, step *Step) *container.HostConfig {
 	if len(step.ExtraHosts) > 0 {
 		config.ExtraHosts = step.ExtraHosts
 	}
-	// if step.Resources != nil {
-	// 	config.Resources = container.Resources{}
-	// 	if limits := step.Resources.Limits; limits != nil {
-	// 		config.Resources.Memory = limits.Memory
-	// 		// TODO(bradrydewski) set config.Resources.CPUPercent
-
-	// 		// IMPORTANT docker and kubernetes use
-	// 		// different units of measure for cpu limits.
-	// 		// we need to figure out how to convert from
-	// 		// the kubernetes unit of measure to the docker
-	// 		// unit of measure.
-	// 	}
-	// }
+	if isUnlimited(step) == false {
+		config.Resources = container.Resources{
+			CPUPeriod:  step.CPUPeriod,
+			CPUQuota:   step.CPUQuota,
+			CpusetCpus: strings.Join(step.CPUSet, ","),
+			CPUShares:  step.CPUShares,
+			Memory:     step.MemLimit,
+			MemorySwap: step.MemSwapLimit,
+		}
+	}
 
 	if len(step.Volumes) != 0 {
 		config.Devices = toDeviceSlice(spec, step)
@@ -259,6 +256,16 @@ func toEnv(env map[string]string) []string {
 	return envs
 }
 
+// returns true if the container has no resource limits.
+func isUnlimited(res *Step) bool {
+	return len(res.CPUSet) == 0 &&
+		res.CPUPeriod == 0 &&
+		res.CPUQuota == 0 &&
+		res.CPUShares == 0 &&
+		res.MemLimit == 0 &&
+		res.MemSwapLimit == 0
+}
+
 // returns true if the volume is a bind mount.
 func isBindMount(volume *Volume) bool {
 	return volume.HostPath != nil

engine/engine_impl.go

@@ -214,19 +214,6 @@ func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.W
 		return err
 	}
 
-	// // use the default user-defined network if network_mode
-	// // is not otherwise specified.
-	// if step.Network == "" {
-	// 	for _, net := range step.Networks {
-	// 		err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{
-	// 			Aliases: []string{net},
-	// 		})
-	// 		if err != nil {
-	// 			return nil
-	// 		}
-	// 	}
-	// }
-
 	return nil
 }
 

engine/resource/pipeline.go

@@ -13,8 +13,6 @@ var (
 	_ manifest.PlatformResource  = (*Pipeline)(nil)
 )
 
-// TODO(bradrydzewski) add resource limits
-
 // Defines the Resource Kind and Type.
 const (
 	Kind = "pipeline"
@@ -108,8 +106,6 @@ type (
 		Volumes     []*VolumeMount                 `json:"volumes,omitempty"`
 		When        manifest.Conditions            `json:"when,omitempty"`
 		WorkingDir  string                         `json:"working_dir,omitempty" yaml:"working_dir"`
-
-		// Resources *Resources `json:"resources,omitempty"`
 	}
 
 	// Volume that can be mounted by containers.

engine/spec.go

@@ -10,7 +10,6 @@ type (
 	// execution.
 	Spec struct {
 		Platform Platform  `json:"platform,omitempty"`
-		Files    []*File   `json:"files,omitempty"`
 		Steps    []*Step   `json:"steps,omitempty"`
 		Volumes  []*Volume `json:"volumes,omitempty"`
 		Network  Network   `json:"network"`
@@ -33,7 +32,6 @@ type (
 		Entrypoint   []string          `json:"entrypoint,omitempty"`
 		Envs         map[string]string `json:"environment,omitempty"`
 		ExtraHosts   []string          `json:"extra_hosts,omitempty"`
-		Files        []*File           `json:"files,omitempty"`
 		IgnoreErr    bool              `json:"ignore_err,omitempty"`
 		IgnoreStdout bool              `json:"ignore_stderr,omitempty"`
 		IgnoreStderr bool              `json:"ignore_stdout,omitempty"`
@@ -54,16 +52,6 @@ type (
 		WorkingDir   string            `json:"working_dir,omitempty"`
 	}
 
-	// File defines a file that should be uploaded or
-	// mounted somewhere in the step container or virtual
-	// machine prior to command execution.
-	File struct {
-		Path  string `json:"path,omitempty"`
-		Mode  uint32 `json:"mode,omitempty"`
-		Data  []byte `json:"data,omitempty"`
-		IsDir bool   `json:"is_dir,omitempty"`
-	}
-
 	// Platform defines the target platform.
 	Platform struct {
 		OS      string `json:"os,omitempty"`