added linter

engine/auth/auth.go

@@ -0,0 +1,122 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Polyform License
+// that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/drone-runners/drone-runner-docker/engine"
+)
+
+// config represents the Docker client configuration,
+// typically located at ~/.docker/config.json
+type config struct {
+	Auths map[string]auths `json:"auths"`
+}
+
+type auths struct {
+	Auth string `json:"auth"`
+}
+
+// Parse parses the registry credential from the reader.
+func Parse(r io.Reader) ([]*engine.Auth, error) {
+	c := new(config)
+	err := json.NewDecoder(r).Decode(c)
+	if err != nil {
+		return nil, err
+	}
+	var auths []*engine.Auth
+	for k, v := range c.Auths {
+		username, password := decode(v.Auth)
+		auths = append(auths, &engine.Auth{
+			Address:  hostname(k),
+			Username: username,
+			Password: password,
+		})
+	}
+	return auths, nil
+}
+
+// ParseFile parses the registry credential file.
+func ParseFile(filepath string) ([]*engine.Auth, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return Parse(f)
+}
+
+// ParseString parses the registry credential file.
+func ParseString(s string) ([]*engine.Auth, error) {
+	return Parse(strings.NewReader(s))
+}
+
+// encode returns the encoded credentials.
+func encode(username, password string) string {
+	return base64.StdEncoding.EncodeToString(
+		[]byte(username + ":" + password),
+	)
+}
+
+// decode returns the decoded credentials.
+func decode(s string) (username, password string) {
+	d, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return
+	}
+	parts := strings.SplitN(string(d), ":", 2)
+	if len(parts) > 0 {
+		username = parts[0]
+	}
+	if len(parts) > 1 {
+		password = parts[1]
+	}
+	return
+}
+
+func hostname(s string) string {
+	uri, _ := url.Parse(s)
+	if uri.Host != "" {
+		s = uri.Host
+	}
+	return s
+}
+
+// Encode returns the json marshaled, base64 encoded
+// credential string that can be passed to the docker
+// registry authentication header.
+func Encode(username, password string) string {
+	v := struct {
+		Username string `json:"username,omitempty"`
+		Password string `json:"password,omitempty"`
+	}{
+		Username: username,
+		Password: password,
+	}
+	buf, _ := json.Marshal(&v)
+	return base64.URLEncoding.EncodeToString(buf)
+}
+
+// Marshal marshals the Auth credentials to a
+// .docker/config.json file.
+func Marshal(list []*engine.Auth) ([]byte, error) {
+	out := &config{}
+	out.Auths = map[string]auths{}
+	for _, item := range list {
+		out.Auths[item.Address] = auths{
+			Auth: encode(
+				item.Username,
+				item.Password,
+			),
+		}
+	}
+	return json.Marshal(out)
+}

engine/auth/auth_test.go

@@ -0,0 +1,143 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Polyform License
+// that can be found in the LICENSE file.
+
+package auth
+
+import (
+	"bytes"
+	"encoding/base64"
+	"os"
+	"testing"
+
+	"github.com/drone-runners/drone-runner-docker/engine"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestParse(t *testing.T) {
+	got, err := ParseString(sample)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	want := []*engine.Auth{
+		{
+			Address:  "index.docker.io",
+			Username: "octocat",
+			Password: "correct-horse-battery-staple",
+		},
+	}
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestParseGCR(t *testing.T) {
+	got, err := ParseFile("testdata/config_gcr.json")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	want := []*engine.Auth{
+		{
+			Address:  "gcr.io",
+			Username: "_json_key",
+			Password: "xxx:bar\n",
+		},
+	}
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestParseErr(t *testing.T) {
+	_, err := ParseString("")
+	if err == nil {
+		t.Errorf("Expect unmarshal error")
+	}
+}
+
+func TestParseFile(t *testing.T) {
+	got, err := ParseFile("./testdata/config.json")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	want := []*engine.Auth{
+		{
+			Address:  "index.docker.io",
+			Username: "octocat",
+			Password: "correct-horse-battery-staple",
+		},
+	}
+	if diff := cmp.Diff(got, want); diff != "" {
+		t.Errorf(diff)
+	}
+}
+
+func TestParseFileErr(t *testing.T) {
+	_, err := ParseFile("./testdata/x.json")
+	if _, ok := err.(*os.PathError); !ok {
+		t.Errorf("Expect error when file does not exist")
+	}
+}
+
+func Test_encodeDecode(t *testing.T) {
+	username := "octocat"
+	password := "correct-horse-battery-staple"
+
+	encoded := encode(username, password)
+	decodedUsername, decodedPassword := decode(encoded)
+	if got, want := decodedUsername, username; got != want {
+		t.Errorf("Want decoded username %s, got %s", want, got)
+	}
+	if got, want := decodedPassword, password; got != want {
+		t.Errorf("Want decoded password %s, got %s", want, got)
+	}
+}
+
+func Test_decodeInvalid(t *testing.T) {
+	username, password := decode("b2N0b2NhdDp==")
+	if username != "" || password != "" {
+		t.Errorf("Expect decoding error")
+	}
+}
+
+func TestEncode(t *testing.T) {
+	username := "octocat"
+	password := "correct-horse-battery-staple"
+	result := Encode(username, password)
+	got, err := base64.URLEncoding.DecodeString(result)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	want := []byte(`{"username":"octocat","password":"correct-horse-battery-staple"}`)
+	if bytes.Equal(got, want) == false {
+		t.Errorf("Could not decode credential header")
+	}
+}
+
+func TestMarshal(t *testing.T) {
+	auths := []*engine.Auth{
+		{
+			Address:  "index.docker.io",
+			Username: "octocat",
+			Password: "correct-horse-battery-staple",
+		},
+	}
+	got, _ := Marshal(auths)
+	want := []byte(`{"auths":{"index.docker.io":{"auth":"b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl"}}}`)
+	if bytes.Equal(got, want) == false {
+		t.Errorf("Could not decode credential header")
+	}
+}
+
+var sample = `{
+	"auths": {
+		"https://index.docker.io/v1/": {
+			"auth": "b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl"
+		}
+	}
+}`

engine/auth/testdata/config.json

@@ -0,0 +1,7 @@
+{
+	"auths": {
+		"https://index.docker.io/v1/": {
+			"auth": "b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl"
+		}
+	}
+}

engine/auth/testdata/config_gcr.json

@@ -0,0 +1,7 @@
+{
+    "auths": {
+        "gcr.io": {
+            "auth": "X2pzb25fa2V5Onh4eDpiYXIK"
+        }
+    }
+}

engine/compiler/compiler.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 
 	"github.com/drone-runners/drone-runner-docker/engine"
+	"github.com/drone-runners/drone-runner-docker/engine/auth"
+	"github.com/drone-runners/drone-runner-docker/engine/compiler/image"
 	"github.com/drone-runners/drone-runner-docker/engine/resource"
 
 	"github.com/drone/drone-go/drone"
@@ -84,13 +86,15 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 
 	// create the workspace volume
 	volume := &engine.VolumeEmptyDir{
-		ID:   random(),
+		ID:     random(),
-		Name: mount.Name,
+		Name:   mount.Name,
+		Labels: createLabels(c.Repo, c.Build, c.Stage, nil),
 	}
 
 	spec := &engine.Spec{
 		Network: engine.Network{
-			ID: random(),
+			ID:     random(),
+			Labels: createLabels(c.Repo, c.Build, c.Stage, nil),
 		},
 		Platform: engine.Platform{
 			OS:      c.Pipeline.Platform.OS,
@@ -107,6 +111,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 	envs := environ.Combine(
 		c.Environ,
 		c.Build.Params,
+		c.Pipeline.Environment,
 		environ.Proxy(),
 		environ.System(c.System),
 		environ.Repo(c.Repo),
@@ -162,6 +167,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 		step.ID = random()
 		step.Envs = environ.Combine(envs, step.Envs)
 		step.WorkingDir = full
+		step.Labels = createLabels(c.Repo, c.Build, c.Stage, nil)
 		step.Volumes = append(step.Volumes, mount)
 		spec.Steps = append(spec.Steps, step)
 	}
@@ -172,6 +178,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 		dst.Detach = true
 		dst.Envs = environ.Combine(envs, dst.Envs)
 		dst.Volumes = append(dst.Volumes, mount)
+		dst.Labels = createLabels(c.Repo, c.Build, c.Stage, nil)
 		setupScript(src, dst, os)
 		setupWorkdir(src, dst, full)
 		spec.Steps = append(spec.Steps, dst)
@@ -188,6 +195,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 		dst := createStep(c.Pipeline, src)
 		dst.Envs = environ.Combine(envs, dst.Envs)
 		dst.Volumes = append(dst.Volumes, mount)
+		dst.Labels = createLabels(c.Repo, c.Build, c.Stage, nil)
 		setupScript(src, dst, full)
 		setupWorkdir(src, dst, full)
 		spec.Steps = append(spec.Steps, dst)
@@ -216,6 +224,27 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec {
 		}
 	}
 
+	var auths []*engine.Auth
+	for _, name := range c.Pipeline.PullSecrets {
+		secret, ok := c.findSecret(ctx, name)
+		if ok {
+			parsed, err := auth.ParseString(secret)
+			if err == nil {
+				auths = append(auths, parsed...)
+			}
+		}
+	}
+
+	for _, step := range spec.Steps {
+	STEPS:
+		for _, auth := range auths {
+			if image.MatchHostname(step.Image, auth.Address) {
+				step.Auth = auth
+				break STEPS
+			}
+		}
+	}
+
 	return spec
 }
 

engine/compiler/label.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Polyform License
+// that can be found in the LICENSE file.
+
+package compiler
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/drone/drone-go/drone"
+)
+
+func createLabels(
+	repo *drone.Repo,
+	build *drone.Build,
+	stage *drone.Stage,
+	step *drone.Step,
+) map[string]string {
+	labels := map[string]string{
+		"io.drone":                "true",
+		"io.drone.build.number":   fmt.Sprint(build.Number),
+		"io.drone.repo.namespace": repo.Namespace,
+		"io.drone.repo.name":      repo.Name,
+		"io.drone.stage.name":     stage.Name,
+		"io.drone.stage.number":   fmt.Sprint(stage.Number),
+		"io.drone.ttl":            fmt.Sprint(time.Duration(repo.Timeout) * time.Minute),
+		"io.drone.expires":        fmt.Sprint(time.Now().Add(time.Duration(repo.Timeout)*time.Minute + time.Hour).Unix()),
+		"io.drone.created":        fmt.Sprint(time.Now().Unix()),
+		"io.drone.protected":      "false",
+	}
+	if step != nil {
+		labels["io.drone.step.name"] = step.Name
+		labels["io.drone.step.number"] = fmt.Sprint(step.Number)
+	}
+	return labels
+}

engine/linter/linter.go

@@ -0,0 +1,153 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Polyform License
+// that can be found in the LICENSE file.
+
+package linter
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/drone-runners/drone-runner-docker/engine/resource"
+)
+
+// ErrDuplicateStepName is returned when two Pipeline steps
+// have the same name.
+var ErrDuplicateStepName = errors.New("linter: duplicate step names")
+
+// ErrMissingDependency is returned when a Pipeline step
+// defines dependencies that are invlid or unknown.
+var ErrMissingDependency = errors.New("linter: invalid or unknown step dependency")
+
+// ErrCyclicalDependency is returned when a Pipeline step
+// defines a cyclical dependency, which would result in an
+// infinite execution loop.
+var ErrCyclicalDependency = errors.New("linter: cyclical step dependency detected")
+
+// Opts provides linting options.
+type Opts struct {
+	Trusted bool
+}
+
+// Linter evaluates the pipeline against a set of
+// rules and returns an error if one or more of the
+// rules are broken.
+type Linter struct{}
+
+// Lint executes the linting rules for the pipeline
+// configuration.
+func (l *Linter) Lint(pipeline *resource.Pipeline, opts Opts) error {
+	return checkPipeline(pipeline, opts.Trusted)
+}
+
+func checkPipeline(pipeline *resource.Pipeline, trusted bool) error {
+	if err := checkNames(pipeline); err != nil {
+		return err
+	}
+	if err := checkSteps(pipeline, trusted); err != nil {
+		return err
+	}
+	if err := checkVolumes(pipeline, trusted); err != nil {
+		return err
+	}
+	return nil
+}
+
+func checkNames(pipeline *resource.Pipeline) error {
+	names := map[string]struct{}{}
+	if !pipeline.Clone.Disable {
+		names["clone"] = struct{}{}
+	}
+	steps := append(pipeline.Services, pipeline.Steps...)
+	for _, step := range steps {
+		_, ok := names[step.Name]
+		if ok {
+			return ErrDuplicateStepName
+		}
+		names[step.Name] = struct{}{}
+	}
+	return nil
+}
+
+func checkSteps(pipeline *resource.Pipeline, trusted bool) error {
+	steps := append(pipeline.Services, pipeline.Steps...)
+	for _, step := range steps {
+		if err := checkStep(step, trusted); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func checkStep(step *resource.Step, trusted bool) error {
+	if step.Image == "" {
+		return errors.New("linter: invalid or missing image")
+	}
+	if step.Name == "" {
+		return errors.New("linter: invalid or missing name")
+	}
+	if len(step.Name) > 100 {
+		return errors.New("linter: name exceeds maximum length")
+	}
+	if trusted == false && step.Privileged {
+		return errors.New("linter: untrusted repositories cannot enable privileged mode")
+	}
+	if trusted == false && len(step.Devices) > 0 {
+		return errors.New("linter: untrusted repositories cannot mount devices")
+	}
+	if trusted == false && len(step.DNS) > 0 {
+		return errors.New("linter: untrusted repositories cannot configure dns")
+	}
+	if trusted == false && len(step.DNSSearch) > 0 {
+		return errors.New("linter: untrusted repositories cannot configure dns_search")
+	}
+	if trusted == false && len(step.ExtraHosts) > 0 {
+		return errors.New("linter: untrusted repositories cannot configure extra_hosts")
+	}
+	if trusted == false && len(step.Network) > 0 {
+		return errors.New("linter: untrusted repositories cannot configure network_mode")
+	}
+	for _, mount := range step.Volumes {
+		switch mount.Name {
+		case "workspace", "_workspace", "_docker_socket":
+			return fmt.Errorf("linter: invalid volume name: %s", mount.Name)
+		}
+	}
+	return nil
+}
+
+func checkVolumes(pipeline *resource.Pipeline, trusted bool) error {
+	for _, volume := range pipeline.Volumes {
+		if volume.EmptyDir != nil {
+			err := checkEmptyDirVolume(volume.EmptyDir, trusted)
+			if err != nil {
+				return err
+			}
+		}
+		if volume.HostPath != nil {
+			err := checkHostPathVolume(volume.HostPath, trusted)
+			if err != nil {
+				return err
+			}
+		}
+		switch volume.Name {
+		case "workspace", "_workspace", "_docker_socket":
+			return fmt.Errorf("linter: invalid volume name: %s", volume.Name)
+		}
+	}
+	return nil
+}
+
+func checkHostPathVolume(volume *resource.VolumeHostPath, trusted bool) error {
+	if trusted == false {
+		return errors.New("linter: untrusted repositories cannot mount host volumes")
+	}
+	return nil
+}
+
+func checkEmptyDirVolume(volume *resource.VolumeEmptyDir, trusted bool) error {
+	if trusted == false && volume.Medium == "memory" {
+		return errors.New("linter: untrusted repositories cannot mount in-memory volumes")
+	}
+	return nil
+}

engine/linter/linter_test.go

@@ -0,0 +1,5 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Polyform License
+// that can be found in the LICENSE file.
+
+package linter

engine/resource/pipeline.go

@@ -36,11 +36,12 @@ type Pipeline struct {
 	Platform    manifest.Platform    `json:"platform,omitempty"`
 	Trigger     manifest.Conditions  `json:"conditions,omitempty"`
 
-	Services    []*Step   `json:"services,omitempty"`
+	Environment map[string]string `json:"environment,omitempty"`
-	Steps       []*Step   `json:"steps,omitempty"`
+	Services    []*Step           `json:"services,omitempty"`
-	Volumes     []*Volume `json:"volumes,omitempty"`
+	Steps       []*Step           `json:"steps,omitempty"`
-	PullSecrets []string  `json:"image_pull_secrets,omitempty" yaml:"image_pull_secrets"`
+	Volumes     []*Volume         `json:"volumes,omitempty"`
-	Workspace   Workspace `json:"workspace,omitempty"`
+	PullSecrets []string          `json:"image_pull_secrets,omitempty" yaml:"image_pull_secrets"`
+	Workspace   Workspace         `json:"workspace,omitempty"`
 }
 
 // GetVersion returns the resource version.

engine/spec.go

@@ -19,6 +19,7 @@ type (
 	// Step defines a pipeline step.
 	Step struct {
 		ID           string            `json:"id,omitempty"`
+		Auth         *Auth             `json:"auth,omitempty"`
 		Command      []string          `json:"args,omitempty"`
 		Detach       bool              `json:"detach,omitempty"`
 		DependsOn    []string          `json:"depends_on,omitempty"`
@@ -32,6 +33,7 @@ type (
 		IgnoreStdout bool              `json:"ignore_stderr,omitempty"`
 		IgnoreStderr bool              `json:"ignore_stdout,omitempty"`
 		Image        string            `json:"image,omitempty"`
+		Labels       map[string]string `json:"labels,omitempty"`
 		Name         string            `json:"name,omitempty"`
 		Network      string            `json:"network,omitempty"`
 		Networks     []string          `json:"networks,omitempty"`
@@ -94,22 +96,32 @@ type (
 	// host node's filesystem into the container. This can
 	// be used as a shared scratch space.
 	VolumeEmptyDir struct {
-		ID        string `json:"id,omitempty"`
+		ID        string            `json:"id,omitempty"`
-		Name      string `json:"name,omitempty"`
+		Name      string            `json:"name,omitempty"`
-		Medium    string `json:"medium,omitempty"`
+		Medium    string            `json:"medium,omitempty"`
-		SizeLimit int64  `json:"size_limit,omitempty"`
+		SizeLimit int64             `json:"size_limit,omitempty"`
+		Labels    map[string]string `json:"labels,omitempty"`
 	}
 
 	// VolumeHostPath mounts a file or directory from the
 	// host node's filesystem into your container.
 	VolumeHostPath struct {
-		ID   string `json:"id,omitempty"`
+		ID     string            `json:"id,omitempty"`
-		Name string `json:"name,omitempty"`
+		Name   string            `json:"name,omitempty"`
-		Path string `json:"path,omitempty"`
+		Path   string            `json:"path,omitempty"`
+		Labels map[string]string `json:"labels,omitempty"`
 	}
 
 	// Network that is created and attached to containers
 	Network struct {
-		ID string `json:"id,omitempty"`
+		ID     string            `json:"id,omitempty"`
+		Labels map[string]string `json:"labels,omitempty"`
+	}
+
+	// Auth defines dockerhub authentication credentials.
+	Auth struct {
+		Address  string `json:"address,omitempty"`
+		Username string `json:"username,omitempty"`
+		Password string `json:"password,omitempty"`
 	}
 )