From 47c1f5248ac17f8734563b7ff7aafce10e50bd9d Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Wed, 16 Oct 2019 23:27:43 -0700 Subject: [PATCH] added linter --- engine/auth/auth.go | 122 +++++++++++++++++++++ engine/auth/auth_test.go | 143 +++++++++++++++++++++++++ engine/auth/testdata/config.json | 7 ++ engine/auth/testdata/config_gcr.json | 7 ++ engine/compiler/compiler.go | 35 +++++- engine/compiler/label.go | 37 +++++++ engine/linter/linter.go | 153 +++++++++++++++++++++++++++ engine/linter/linter_test.go | 5 + engine/resource/pipeline.go | 11 +- engine/spec.go | 28 +++-- 10 files changed, 532 insertions(+), 16 deletions(-) create mode 100644 engine/auth/auth.go create mode 100644 engine/auth/auth_test.go create mode 100644 engine/auth/testdata/config.json create mode 100644 engine/auth/testdata/config_gcr.json create mode 100644 engine/compiler/label.go create mode 100644 engine/linter/linter.go create mode 100644 engine/linter/linter_test.go diff --git a/engine/auth/auth.go b/engine/auth/auth.go new file mode 100644 index 0000000..997bb12 --- /dev/null +++ b/engine/auth/auth.go @@ -0,0 +1,122 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package auth + +import ( + "encoding/base64" + "encoding/json" + "io" + "net/url" + "os" + "strings" + + "github.com/drone-runners/drone-runner-docker/engine" +) + +// config represents the Docker client configuration, +// typically located at ~/.docker/config.json +type config struct { + Auths map[string]auths `json:"auths"` +} + +type auths struct { + Auth string `json:"auth"` +} + +// Parse parses the registry credential from the reader. +func Parse(r io.Reader) ([]*engine.Auth, error) { + c := new(config) + err := json.NewDecoder(r).Decode(c) + if err != nil { + return nil, err + } + var auths []*engine.Auth + for k, v := range c.Auths { + username, password := decode(v.Auth) + auths = append(auths, &engine.Auth{ + Address: hostname(k), + Username: username, + Password: password, + }) + } + return auths, nil +} + +// ParseFile parses the registry credential file. +func ParseFile(filepath string) ([]*engine.Auth, error) { + f, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer f.Close() + return Parse(f) +} + +// ParseString parses the registry credential file. +func ParseString(s string) ([]*engine.Auth, error) { + return Parse(strings.NewReader(s)) +} + +// encode returns the encoded credentials. +func encode(username, password string) string { + return base64.StdEncoding.EncodeToString( + []byte(username + ":" + password), + ) +} + +// decode returns the decoded credentials. +func decode(s string) (username, password string) { + d, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return + } + parts := strings.SplitN(string(d), ":", 2) + if len(parts) > 0 { + username = parts[0] + } + if len(parts) > 1 { + password = parts[1] + } + return +} + +func hostname(s string) string { + uri, _ := url.Parse(s) + if uri.Host != "" { + s = uri.Host + } + return s +} + +// Encode returns the json marshaled, base64 encoded +// credential string that can be passed to the docker +// registry authentication header. +func Encode(username, password string) string { + v := struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + }{ + Username: username, + Password: password, + } + buf, _ := json.Marshal(&v) + return base64.URLEncoding.EncodeToString(buf) +} + +// Marshal marshals the Auth credentials to a +// .docker/config.json file. +func Marshal(list []*engine.Auth) ([]byte, error) { + out := &config{} + out.Auths = map[string]auths{} + for _, item := range list { + out.Auths[item.Address] = auths{ + Auth: encode( + item.Username, + item.Password, + ), + } + } + return json.Marshal(out) +} diff --git a/engine/auth/auth_test.go b/engine/auth/auth_test.go new file mode 100644 index 0000000..6b94d72 --- /dev/null +++ b/engine/auth/auth_test.go @@ -0,0 +1,143 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package auth + +import ( + "bytes" + "encoding/base64" + "os" + "testing" + + "github.com/drone-runners/drone-runner-docker/engine" + + "github.com/google/go-cmp/cmp" +) + +func TestParse(t *testing.T) { + got, err := ParseString(sample) + if err != nil { + t.Error(err) + return + } + want := []*engine.Auth{ + { + Address: "index.docker.io", + Username: "octocat", + Password: "correct-horse-battery-staple", + }, + } + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf(diff) + } +} + +func TestParseGCR(t *testing.T) { + got, err := ParseFile("testdata/config_gcr.json") + if err != nil { + t.Error(err) + return + } + want := []*engine.Auth{ + { + Address: "gcr.io", + Username: "_json_key", + Password: "xxx:bar\n", + }, + } + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf(diff) + } +} + +func TestParseErr(t *testing.T) { + _, err := ParseString("") + if err == nil { + t.Errorf("Expect unmarshal error") + } +} + +func TestParseFile(t *testing.T) { + got, err := ParseFile("./testdata/config.json") + if err != nil { + t.Error(err) + return + } + want := []*engine.Auth{ + { + Address: "index.docker.io", + Username: "octocat", + Password: "correct-horse-battery-staple", + }, + } + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf(diff) + } +} + +func TestParseFileErr(t *testing.T) { + _, err := ParseFile("./testdata/x.json") + if _, ok := err.(*os.PathError); !ok { + t.Errorf("Expect error when file does not exist") + } +} + +func Test_encodeDecode(t *testing.T) { + username := "octocat" + password := "correct-horse-battery-staple" + + encoded := encode(username, password) + decodedUsername, decodedPassword := decode(encoded) + if got, want := decodedUsername, username; got != want { + t.Errorf("Want decoded username %s, got %s", want, got) + } + if got, want := decodedPassword, password; got != want { + t.Errorf("Want decoded password %s, got %s", want, got) + } +} + +func Test_decodeInvalid(t *testing.T) { + username, password := decode("b2N0b2NhdDp==") + if username != "" || password != "" { + t.Errorf("Expect decoding error") + } +} + +func TestEncode(t *testing.T) { + username := "octocat" + password := "correct-horse-battery-staple" + result := Encode(username, password) + got, err := base64.URLEncoding.DecodeString(result) + if err != nil { + t.Error(err) + return + } + want := []byte(`{"username":"octocat","password":"correct-horse-battery-staple"}`) + if bytes.Equal(got, want) == false { + t.Errorf("Could not decode credential header") + } +} + +func TestMarshal(t *testing.T) { + auths := []*engine.Auth{ + { + Address: "index.docker.io", + Username: "octocat", + Password: "correct-horse-battery-staple", + }, + } + got, _ := Marshal(auths) + want := []byte(`{"auths":{"index.docker.io":{"auth":"b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl"}}}`) + if bytes.Equal(got, want) == false { + t.Errorf("Could not decode credential header") + } +} + +var sample = `{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl" + } + } +}` diff --git a/engine/auth/testdata/config.json b/engine/auth/testdata/config.json new file mode 100644 index 0000000..382c690 --- /dev/null +++ b/engine/auth/testdata/config.json @@ -0,0 +1,7 @@ +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "b2N0b2NhdDpjb3JyZWN0LWhvcnNlLWJhdHRlcnktc3RhcGxl" + } + } +} \ No newline at end of file diff --git a/engine/auth/testdata/config_gcr.json b/engine/auth/testdata/config_gcr.json new file mode 100644 index 0000000..259907a --- /dev/null +++ b/engine/auth/testdata/config_gcr.json @@ -0,0 +1,7 @@ +{ + "auths": { + "gcr.io": { + "auth": "X2pzb25fa2V5Onh4eDpiYXIK" + } + } +} \ No newline at end of file diff --git a/engine/compiler/compiler.go b/engine/compiler/compiler.go index 6274f5b..1549568 100644 --- a/engine/compiler/compiler.go +++ b/engine/compiler/compiler.go @@ -9,6 +9,8 @@ import ( "fmt" "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/auth" + "github.com/drone-runners/drone-runner-docker/engine/compiler/image" "github.com/drone-runners/drone-runner-docker/engine/resource" "github.com/drone/drone-go/drone" @@ -84,13 +86,15 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { // create the workspace volume volume := &engine.VolumeEmptyDir{ - ID: random(), - Name: mount.Name, + ID: random(), + Name: mount.Name, + Labels: createLabels(c.Repo, c.Build, c.Stage, nil), } spec := &engine.Spec{ Network: engine.Network{ - ID: random(), + ID: random(), + Labels: createLabels(c.Repo, c.Build, c.Stage, nil), }, Platform: engine.Platform{ OS: c.Pipeline.Platform.OS, @@ -107,6 +111,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { envs := environ.Combine( c.Environ, c.Build.Params, + c.Pipeline.Environment, environ.Proxy(), environ.System(c.System), environ.Repo(c.Repo), @@ -162,6 +167,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { step.ID = random() step.Envs = environ.Combine(envs, step.Envs) step.WorkingDir = full + step.Labels = createLabels(c.Repo, c.Build, c.Stage, nil) step.Volumes = append(step.Volumes, mount) spec.Steps = append(spec.Steps, step) } @@ -172,6 +178,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { dst.Detach = true dst.Envs = environ.Combine(envs, dst.Envs) dst.Volumes = append(dst.Volumes, mount) + dst.Labels = createLabels(c.Repo, c.Build, c.Stage, nil) setupScript(src, dst, os) setupWorkdir(src, dst, full) spec.Steps = append(spec.Steps, dst) @@ -188,6 +195,7 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { dst := createStep(c.Pipeline, src) dst.Envs = environ.Combine(envs, dst.Envs) dst.Volumes = append(dst.Volumes, mount) + dst.Labels = createLabels(c.Repo, c.Build, c.Stage, nil) setupScript(src, dst, full) setupWorkdir(src, dst, full) spec.Steps = append(spec.Steps, dst) @@ -216,6 +224,27 @@ func (c *Compiler) Compile(ctx context.Context) *engine.Spec { } } + var auths []*engine.Auth + for _, name := range c.Pipeline.PullSecrets { + secret, ok := c.findSecret(ctx, name) + if ok { + parsed, err := auth.ParseString(secret) + if err == nil { + auths = append(auths, parsed...) + } + } + } + + for _, step := range spec.Steps { + STEPS: + for _, auth := range auths { + if image.MatchHostname(step.Image, auth.Address) { + step.Auth = auth + break STEPS + } + } + } + return spec } diff --git a/engine/compiler/label.go b/engine/compiler/label.go new file mode 100644 index 0000000..b1666b7 --- /dev/null +++ b/engine/compiler/label.go @@ -0,0 +1,37 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "fmt" + "time" + + "github.com/drone/drone-go/drone" +) + +func createLabels( + repo *drone.Repo, + build *drone.Build, + stage *drone.Stage, + step *drone.Step, +) map[string]string { + labels := map[string]string{ + "io.drone": "true", + "io.drone.build.number": fmt.Sprint(build.Number), + "io.drone.repo.namespace": repo.Namespace, + "io.drone.repo.name": repo.Name, + "io.drone.stage.name": stage.Name, + "io.drone.stage.number": fmt.Sprint(stage.Number), + "io.drone.ttl": fmt.Sprint(time.Duration(repo.Timeout) * time.Minute), + "io.drone.expires": fmt.Sprint(time.Now().Add(time.Duration(repo.Timeout)*time.Minute + time.Hour).Unix()), + "io.drone.created": fmt.Sprint(time.Now().Unix()), + "io.drone.protected": "false", + } + if step != nil { + labels["io.drone.step.name"] = step.Name + labels["io.drone.step.number"] = fmt.Sprint(step.Number) + } + return labels +} diff --git a/engine/linter/linter.go b/engine/linter/linter.go new file mode 100644 index 0000000..98e2e37 --- /dev/null +++ b/engine/linter/linter.go @@ -0,0 +1,153 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package linter + +import ( + "errors" + "fmt" + + "github.com/drone-runners/drone-runner-docker/engine/resource" +) + +// ErrDuplicateStepName is returned when two Pipeline steps +// have the same name. +var ErrDuplicateStepName = errors.New("linter: duplicate step names") + +// ErrMissingDependency is returned when a Pipeline step +// defines dependencies that are invlid or unknown. +var ErrMissingDependency = errors.New("linter: invalid or unknown step dependency") + +// ErrCyclicalDependency is returned when a Pipeline step +// defines a cyclical dependency, which would result in an +// infinite execution loop. +var ErrCyclicalDependency = errors.New("linter: cyclical step dependency detected") + +// Opts provides linting options. +type Opts struct { + Trusted bool +} + +// Linter evaluates the pipeline against a set of +// rules and returns an error if one or more of the +// rules are broken. +type Linter struct{} + +// Lint executes the linting rules for the pipeline +// configuration. +func (l *Linter) Lint(pipeline *resource.Pipeline, opts Opts) error { + return checkPipeline(pipeline, opts.Trusted) +} + +func checkPipeline(pipeline *resource.Pipeline, trusted bool) error { + if err := checkNames(pipeline); err != nil { + return err + } + if err := checkSteps(pipeline, trusted); err != nil { + return err + } + if err := checkVolumes(pipeline, trusted); err != nil { + return err + } + return nil +} + +func checkNames(pipeline *resource.Pipeline) error { + names := map[string]struct{}{} + if !pipeline.Clone.Disable { + names["clone"] = struct{}{} + } + steps := append(pipeline.Services, pipeline.Steps...) + for _, step := range steps { + _, ok := names[step.Name] + if ok { + return ErrDuplicateStepName + } + names[step.Name] = struct{}{} + } + return nil +} + +func checkSteps(pipeline *resource.Pipeline, trusted bool) error { + steps := append(pipeline.Services, pipeline.Steps...) + for _, step := range steps { + if err := checkStep(step, trusted); err != nil { + return err + } + } + return nil +} + +func checkStep(step *resource.Step, trusted bool) error { + if step.Image == "" { + return errors.New("linter: invalid or missing image") + } + if step.Name == "" { + return errors.New("linter: invalid or missing name") + } + if len(step.Name) > 100 { + return errors.New("linter: name exceeds maximum length") + } + if trusted == false && step.Privileged { + return errors.New("linter: untrusted repositories cannot enable privileged mode") + } + if trusted == false && len(step.Devices) > 0 { + return errors.New("linter: untrusted repositories cannot mount devices") + } + if trusted == false && len(step.DNS) > 0 { + return errors.New("linter: untrusted repositories cannot configure dns") + } + if trusted == false && len(step.DNSSearch) > 0 { + return errors.New("linter: untrusted repositories cannot configure dns_search") + } + if trusted == false && len(step.ExtraHosts) > 0 { + return errors.New("linter: untrusted repositories cannot configure extra_hosts") + } + if trusted == false && len(step.Network) > 0 { + return errors.New("linter: untrusted repositories cannot configure network_mode") + } + for _, mount := range step.Volumes { + switch mount.Name { + case "workspace", "_workspace", "_docker_socket": + return fmt.Errorf("linter: invalid volume name: %s", mount.Name) + } + } + return nil +} + +func checkVolumes(pipeline *resource.Pipeline, trusted bool) error { + for _, volume := range pipeline.Volumes { + if volume.EmptyDir != nil { + err := checkEmptyDirVolume(volume.EmptyDir, trusted) + if err != nil { + return err + } + } + if volume.HostPath != nil { + err := checkHostPathVolume(volume.HostPath, trusted) + if err != nil { + return err + } + } + switch volume.Name { + case "workspace", "_workspace", "_docker_socket": + return fmt.Errorf("linter: invalid volume name: %s", volume.Name) + } + } + return nil +} + +func checkHostPathVolume(volume *resource.VolumeHostPath, trusted bool) error { + if trusted == false { + return errors.New("linter: untrusted repositories cannot mount host volumes") + } + return nil +} + +func checkEmptyDirVolume(volume *resource.VolumeEmptyDir, trusted bool) error { + if trusted == false && volume.Medium == "memory" { + return errors.New("linter: untrusted repositories cannot mount in-memory volumes") + } + return nil +} diff --git a/engine/linter/linter_test.go b/engine/linter/linter_test.go new file mode 100644 index 0000000..921f13a --- /dev/null +++ b/engine/linter/linter_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package linter diff --git a/engine/resource/pipeline.go b/engine/resource/pipeline.go index 7fd881e..030ce74 100644 --- a/engine/resource/pipeline.go +++ b/engine/resource/pipeline.go @@ -36,11 +36,12 @@ type Pipeline struct { Platform manifest.Platform `json:"platform,omitempty"` Trigger manifest.Conditions `json:"conditions,omitempty"` - Services []*Step `json:"services,omitempty"` - Steps []*Step `json:"steps,omitempty"` - Volumes []*Volume `json:"volumes,omitempty"` - PullSecrets []string `json:"image_pull_secrets,omitempty" yaml:"image_pull_secrets"` - Workspace Workspace `json:"workspace,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Services []*Step `json:"services,omitempty"` + Steps []*Step `json:"steps,omitempty"` + Volumes []*Volume `json:"volumes,omitempty"` + PullSecrets []string `json:"image_pull_secrets,omitempty" yaml:"image_pull_secrets"` + Workspace Workspace `json:"workspace,omitempty"` } // GetVersion returns the resource version. diff --git a/engine/spec.go b/engine/spec.go index 0e5fa4c..d80a33b 100644 --- a/engine/spec.go +++ b/engine/spec.go @@ -19,6 +19,7 @@ type ( // Step defines a pipeline step. Step struct { ID string `json:"id,omitempty"` + Auth *Auth `json:"auth,omitempty"` Command []string `json:"args,omitempty"` Detach bool `json:"detach,omitempty"` DependsOn []string `json:"depends_on,omitempty"` @@ -32,6 +33,7 @@ type ( IgnoreStdout bool `json:"ignore_stderr,omitempty"` IgnoreStderr bool `json:"ignore_stdout,omitempty"` Image string `json:"image,omitempty"` + Labels map[string]string `json:"labels,omitempty"` Name string `json:"name,omitempty"` Network string `json:"network,omitempty"` Networks []string `json:"networks,omitempty"` @@ -94,22 +96,32 @@ type ( // host node's filesystem into the container. This can // be used as a shared scratch space. VolumeEmptyDir struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Medium string `json:"medium,omitempty"` - SizeLimit int64 `json:"size_limit,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Medium string `json:"medium,omitempty"` + SizeLimit int64 `json:"size_limit,omitempty"` + Labels map[string]string `json:"labels,omitempty"` } // VolumeHostPath mounts a file or directory from the // host node's filesystem into your container. VolumeHostPath struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Path string `json:"path,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + Labels map[string]string `json:"labels,omitempty"` } // Network that is created and attached to containers Network struct { - ID string `json:"id,omitempty"` + ID string `json:"id,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + } + + // Auth defines dockerhub authentication credentials. + Auth struct { + Address string `json:"address,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` } )