From 2e48cd1b3bdb94e8a45aabce4730d24292a07432 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Fri, 6 Dec 2019 16:10:27 -0800 Subject: [PATCH] abstract polling and execution to runner-go library --- CHANGELOG.md | 1 + command/compile.go | 6 +- command/daemon/daemon.go | 126 ++++++------- command/exec.go | 21 ++- engine/compiler/clone.go | 3 +- engine/compiler/clone_test.go | 5 +- engine/compiler/compiler.go | 93 +++------- engine/compiler/compiler_test.go | 9 +- engine/compiler/step.go | 6 +- engine/const.go | 55 ------ engine/const_test.go | 115 ------------ engine/engine.go | 293 +++++++++++++++++++++++++++++-- engine/engine_impl.go | 286 ------------------------------ engine/linter/linter.go | 6 +- engine/linter/linter_test.go | 3 +- engine/replacer/replacer.go | 56 ------ engine/replacer/replacer_test.go | 56 ------ engine/resource/lookup.go | 2 +- engine/spec.go | 62 +++++-- go.mod | 8 +- go.sum | 42 +++++ runtime/execer.go | 242 ------------------------- runtime/execer_test.go | 39 ---- runtime/poller.go | 76 -------- runtime/poller_test.go | 27 --- runtime/runner.go | 249 -------------------------- runtime/runner_test.go | 7 - 27 files changed, 500 insertions(+), 1394 deletions(-) delete mode 100644 engine/engine_impl.go delete mode 100644 engine/replacer/replacer.go delete mode 100644 engine/replacer/replacer_test.go delete mode 100644 runtime/execer.go delete mode 100644 runtime/execer_test.go delete mode 100644 runtime/poller.go delete mode 100644 runtime/poller_test.go delete mode 100644 runtime/runner.go delete mode 100644 runtime/runner_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index eacfd37..f197bf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Changed +- abstract polling and execution to runner-go library - use trace level logging for context errors - prefix docker resource names diff --git a/command/compile.go b/command/compile.go index ecd4782..d9880b9 100644 --- a/command/compile.go +++ b/command/compile.go @@ -19,6 +19,7 @@ import ( "github.com/drone/runner-go/environ" "github.com/drone/runner-go/environ/provider" "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline/runtime" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/secret" @@ -90,8 +91,7 @@ func (c *compileCommand) run(*kingpin.ParseContext) error { // lint the pipeline and return an error if any // linting rules are broken lint := linter.New() - opts := linter.Opts{Trusted: c.Repo.Trusted} - err = lint.Lint(resource, opts) + err = lint.Lint(resource, c.Repo) if err != nil { return err } @@ -117,7 +117,7 @@ func (c *compileCommand) run(*kingpin.ParseContext) error { comp.Mount, _ = os.Getwd() } - args := compiler.Args{ + args := runtime.CompilerArgs{ Pipeline: resource, Manifest: manifest, Build: c.Build, diff --git a/command/daemon/daemon.go b/command/daemon/daemon.go index a4a6475..9d1173f 100644 --- a/command/daemon/daemon.go +++ b/command/daemon/daemon.go @@ -13,15 +13,16 @@ import ( "github.com/drone-runners/drone-runner-docker/engine/linter" "github.com/drone-runners/drone-runner-docker/engine/resource" "github.com/drone-runners/drone-runner-docker/internal/match" - "github.com/drone-runners/drone-runner-docker/runtime" "github.com/drone/runner-go/client" "github.com/drone/runner-go/environ/provider" "github.com/drone/runner-go/handler/router" "github.com/drone/runner-go/logger" loghistory "github.com/drone/runner-go/logger/history" - "github.com/drone/runner-go/pipeline/history" - "github.com/drone/runner-go/pipeline/remote" + "github.com/drone/runner-go/pipeline/reporter/history" + "github.com/drone/runner-go/pipeline/reporter/remote" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/poller" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/secret" "github.com/drone/runner-go/server" @@ -107,67 +108,70 @@ func (c *daemonCommand) run(*kingpin.ParseContext) error { hook := loghistory.New() logrus.AddHook(hook) - poller := &runtime.Poller{ - Client: cli, - Runner: &runtime.Runner{ - Client: cli, - Machine: config.Runner.Name, - Reporter: tracer, - Linter: linter.New(), - Match: match.Func( - config.Limit.Repos, - config.Limit.Events, - config.Limit.Trusted, - ), - Compiler: &compiler.Compiler{ - Clone: config.Runner.Clone, - Privileged: append(config.Runner.Privileged, compiler.Privileged...), - Networks: config.Runner.Networks, - Volumes: config.Runner.Volumes, - Resources: compiler.Resources{ - Memory: config.Resources.Memory, - MemorySwap: config.Resources.MemorySwap, - CPUQuota: config.Resources.CPUQuota, - CPUPeriod: config.Resources.CPUPeriod, - CPUShares: config.Resources.CPUShares, - CPUSet: config.Resources.CPUSet, - }, - Environ: provider.Combine( - provider.Static(config.Runner.Environ), - provider.External( - config.Environ.Endpoint, - config.Environ.Token, - config.Environ.SkipVerify, - ), - ), - Registry: registry.Combine( - registry.File( - config.Docker.Config, - ), - registry.External( - config.Registry.Endpoint, - config.Registry.Token, - config.Registry.SkipVerify, - ), - ), - Secret: secret.Combine( - secret.StaticVars( - config.Runner.Secrets, - ), - secret.External( - config.Secret.Endpoint, - config.Secret.Token, - config.Secret.SkipVerify, - ), - ), + runner := &runtime.Runner{ + Client: cli, + Machine: config.Runner.Name, + Reporter: tracer, + Lookup: resource.Lookup, + Lint: linter.New().Lint, + Match: match.Func( + config.Limit.Repos, + config.Limit.Events, + config.Limit.Trusted, + ), + Compiler: &compiler.Compiler{ + Clone: config.Runner.Clone, + Privileged: append(config.Runner.Privileged, compiler.Privileged...), + Networks: config.Runner.Networks, + Volumes: config.Runner.Volumes, + Resources: compiler.Resources{ + Memory: config.Resources.Memory, + MemorySwap: config.Resources.MemorySwap, + CPUQuota: config.Resources.CPUQuota, + CPUPeriod: config.Resources.CPUPeriod, + CPUShares: config.Resources.CPUShares, + CPUSet: config.Resources.CPUSet, }, - Execer: runtime.NewExecer( - tracer, - remote, - engine, - config.Runner.Procs, + Environ: provider.Combine( + provider.Static(config.Runner.Environ), + provider.External( + config.Environ.Endpoint, + config.Environ.Token, + config.Environ.SkipVerify, + ), + ), + Registry: registry.Combine( + registry.File( + config.Docker.Config, + ), + registry.External( + config.Registry.Endpoint, + config.Registry.Token, + config.Registry.SkipVerify, + ), + ), + Secret: secret.Combine( + secret.StaticVars( + config.Runner.Secrets, + ), + secret.External( + config.Secret.Endpoint, + config.Secret.Token, + config.Secret.SkipVerify, + ), ), }, + Exec: runtime.NewExecer( + tracer, + remote, + engine, + config.Runner.Procs, + ).Exec, + } + + poller := &poller.Poller{ + Client: cli, + Dispatch: runner.Run, Filter: &client.Filter{ Kind: resource.Kind, Type: resource.Type, diff --git a/command/exec.go b/command/exec.go index 6447807..cea970e 100644 --- a/command/exec.go +++ b/command/exec.go @@ -18,7 +18,6 @@ import ( "github.com/drone-runners/drone-runner-docker/engine/compiler" "github.com/drone-runners/drone-runner-docker/engine/linter" "github.com/drone-runners/drone-runner-docker/engine/resource" - "github.com/drone-runners/drone-runner-docker/runtime" "github.com/drone/drone-go/drone" "github.com/drone/envsubst" "github.com/drone/runner-go/environ" @@ -26,7 +25,8 @@ import ( "github.com/drone/runner-go/logger" "github.com/drone/runner-go/manifest" "github.com/drone/runner-go/pipeline" - "github.com/drone/runner-go/pipeline/console" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/pipeline/streamer/console" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/secret" "github.com/drone/signal" @@ -102,7 +102,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { // a configuration can contain multiple pipelines. // get a specific pipeline resource for execution. - resource, err := resource.Lookup(c.Stage.Name, manifest) + res, err := resource.Lookup(c.Stage.Name, manifest) if err != nil { return err } @@ -110,8 +110,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { // lint the pipeline and return an error if any // linting rules are broken lint := linter.New() - opts := linter.Opts{Trusted: c.Repo.Trusted} - err = lint.Lint(resource, opts) + err = lint.Lint(res, c.Repo) if err != nil { return err } @@ -137,8 +136,8 @@ func (c *execCommand) run(*kingpin.ParseContext) error { comp.Mount, _ = os.Getwd() } - args := compiler.Args{ - Pipeline: resource, + args := runtime.CompilerArgs{ + Pipeline: res, Manifest: manifest, Build: c.Build, Netrc: c.Netrc, @@ -146,7 +145,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { Stage: c.Stage, System: c.System, } - spec := comp.Compile(nocontext, args) + spec := comp.Compile(nocontext, args).(*engine.Spec) // include only steps that are in the include list, // if the list in non-empty. @@ -161,7 +160,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { continue I } } - step.RunPolicy = engine.RunNever + step.RunPolicy = runtime.RunNever } } @@ -175,7 +174,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { } for _, name := range c.Exclude { if step.Name == name { - step.RunPolicy = engine.RunNever + step.RunPolicy = runtime.RunNever continue E } } @@ -184,7 +183,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error { // create a step object for each pipeline step. for _, step := range spec.Steps { - if step.RunPolicy == engine.RunNever { + if step.RunPolicy == runtime.RunNever { continue } c.Stage.Steps = append(c.Stage.Steps, &drone.Step{ diff --git a/engine/compiler/clone.go b/engine/compiler/clone.go index 65730f0..f03439e 100644 --- a/engine/compiler/clone.go +++ b/engine/compiler/clone.go @@ -10,6 +10,7 @@ import ( "github.com/drone-runners/drone-runner-docker/engine" "github.com/drone-runners/drone-runner-docker/engine/resource" "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline/runtime" ) // default name of the clone step. @@ -47,7 +48,7 @@ func createClone(src *resource.Pipeline) *engine.Step { return &engine.Step{ Name: cloneStepName, Image: cloneImage(src.Platform), - RunPolicy: engine.RunAlways, + RunPolicy: runtime.RunAlways, Envs: cloneParams(src.Clone), } } diff --git a/engine/compiler/clone_test.go b/engine/compiler/clone_test.go index eb0832d..e3cfd28 100644 --- a/engine/compiler/clone_test.go +++ b/engine/compiler/clone_test.go @@ -13,6 +13,7 @@ import ( "github.com/drone/drone-go/drone" "github.com/drone/runner-go/environ/provider" "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline/runtime" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/secret" @@ -46,7 +47,7 @@ func TestClone(t *testing.T) { Image: "drone/git:latest", Name: "clone", Pull: engine.PullIfNotExists, - RunPolicy: engine.RunAlways, + RunPolicy: runtime.RunAlways, WorkingDir: "/drone/src", Volumes: []*engine.VolumeMount{ &engine.VolumeMount{ @@ -88,7 +89,7 @@ func TestCloneCreate(t *testing.T) { want := &engine.Step{ Name: "clone", Image: "drone/git:latest", - RunPolicy: engine.RunAlways, + RunPolicy: runtime.RunAlways, Envs: map[string]string{"PLUGIN_DEPTH": "50"}, } src := &resource.Pipeline{Clone: manifest.Clone{Depth: 50}} diff --git a/engine/compiler/compiler.go b/engine/compiler/compiler.go index b21f2c3..ecf3955 100644 --- a/engine/compiler/compiler.go +++ b/engine/compiler/compiler.go @@ -12,12 +12,12 @@ import ( "github.com/drone-runners/drone-runner-docker/engine/resource" "github.com/drone-runners/drone-runner-docker/internal/docker/image" - "github.com/drone/drone-go/drone" "github.com/drone/runner-go/clone" "github.com/drone/runner-go/environ" "github.com/drone/runner-go/environ/provider" "github.com/drone/runner-go/labels" "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline/runtime" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/registry/auths" "github.com/drone/runner-go/secret" @@ -52,51 +52,9 @@ type Resources struct { CPUSet []string } -// Args provides compiler arguments. -type Args struct { - // Manifest provides the parsed manifest. - Manifest *manifest.Manifest - - // Pipeline provides the parsed pipeline. This pipeline is - // the compiler source and is converted to the intermediate - // representation by the Compile method. - Pipeline *resource.Pipeline - - // Build provides the compiler with stage information that - // is converted to environment variable format and passed to - // each pipeline step. It is also used to clone the commit. - Build *drone.Build - - // Stage provides the compiler with stage information that - // is converted to environment variable format and passed to - // each pipeline step. - Stage *drone.Stage - - // Repo provides the compiler with repo information. This - // repo information is converted to environment variable - // format and passed to each pipeline step. It is also used - // to clone the repository. - Repo *drone.Repo - - // System provides the compiler with system information that - // is converted to environment variable format and passed to - // each pipeline step. - System *drone.System - - // Netrc provides netrc parameters that can be used by the - // default clone step to authenticate to the remote - // repository. - Netrc *drone.Netrc - - // Secret returns a named secret value that can be injected - // into the pipeline step. - Secret secret.Provider -} - // Compiler compiles the Yaml configuration file to an // intermediate representation optimized for simple execution. type Compiler struct { - // Environ provides a set of environment variables that // should be added to each pipeline step by default. Environ provider.Provider @@ -139,11 +97,12 @@ type Compiler struct { } // Compile compiles the configuration file. -func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { - os := args.Pipeline.Platform.OS +func (c *Compiler) Compile(ctx context.Context, args runtime.CompilerArgs) runtime.Spec { + pipeline := args.Pipeline.(*resource.Pipeline) + os := pipeline.Platform.OS // create the workspace paths - base, path, full := createWorkspace(args.Pipeline) + base, path, full := createWorkspace(pipeline) // if the source code is mounted from the host, the // target mount path inside the container must be the @@ -197,10 +156,10 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { Labels: labels, }, Platform: engine.Platform{ - OS: args.Pipeline.Platform.OS, - Arch: args.Pipeline.Platform.Arch, - Variant: args.Pipeline.Platform.Variant, - Version: args.Pipeline.Platform.Version, + OS: pipeline.Platform.OS, + Arch: pipeline.Platform.Arch, + Variant: pipeline.Platform.Variant, + Version: pipeline.Platform.Version, }, Volumes: []*engine.Volume{volume}, } @@ -215,7 +174,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { envs = environ.Combine( envs, args.Build.Params, - args.Pipeline.Environment, + pipeline.Environment, environ.Proxy(), environ.System(args.System), environ.Repo(args.Repo), @@ -223,8 +182,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { environ.Stage(args.Stage), environ.Link(args.Repo, args.Build, args.System), clone.Environ(clone.Config{ - SkipVerify: args.Pipeline.Clone.SkipVerify, - Trace: args.Pipeline.Clone.Trace, + SkipVerify: pipeline.Clone.SkipVerify, + Trace: pipeline.Clone.Trace, User: clone.User{ Name: args.Build.AuthorName, Email: args.Build.AuthorEmail, @@ -272,8 +231,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { } // create the clone step - if args.Pipeline.Clone.Disable == false { - step := createClone(args.Pipeline) + if pipeline.Clone.Disable == false { + step := createClone(pipeline) step.ID = random() step.Envs = environ.Combine(envs, step.Envs) step.WorkingDir = full @@ -291,13 +250,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { // if the repository is mounted from a local // volume we should disable cloning. if c.Mount != "" { - step.RunPolicy = engine.RunNever + step.RunPolicy = runtime.RunNever } } // create steps - for _, src := range args.Pipeline.Services { - dst := createStep(args.Pipeline, src) + for _, src := range pipeline.Services { + dst := createStep(pipeline, src) dst.Detach = true dst.Envs = environ.Combine(envs, dst.Envs) dst.Volumes = append(dst.Volumes, mount) @@ -309,13 +268,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { // if the pipeline step has unmet conditions the step is // automatically skipped. if !src.When.Match(match) { - dst.RunPolicy = engine.RunNever + dst.RunPolicy = runtime.RunNever } } // create steps - for _, src := range args.Pipeline.Steps { - dst := createStep(args.Pipeline, src) + for _, src := range pipeline.Steps { + dst := createStep(pipeline, src) dst.Envs = environ.Combine(envs, dst.Envs) dst.Volumes = append(dst.Volumes, mount) dst.Labels = labels @@ -326,7 +285,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { // if the pipeline step has unmet conditions the step is // automatically skipped. if !src.When.Match(match) { - dst.RunPolicy = engine.RunNever + dst.RunPolicy = runtime.RunNever } // if the pipeline step has an approved image, it is @@ -339,9 +298,9 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { if isGraph(spec) == false { configureSerial(spec) - } else if args.Pipeline.Clone.Disable == false { + } else if pipeline.Clone.Disable == false { configureCloneDeps(spec) - } else if args.Pipeline.Clone.Disable == true { + } else if pipeline.Clone.Disable == true { removeCloneDeps(spec) } @@ -365,7 +324,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { } // get registry credentials from secrets - for _, name := range args.Pipeline.PullSecrets { + for _, name := range pipeline.PullSecrets { secret, ok := c.findSecret(ctx, args, name) if ok { parsed, err := auths.ParseString(secret) @@ -425,7 +384,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec { } // append volumes - for _, v := range args.Pipeline.Volumes { + for _, v := range pipeline.Volumes { id := random() src := new(engine.Volume) if v.EmptyDir != nil { @@ -478,7 +437,7 @@ func (c *Compiler) isPrivileged(step *resource.Step) bool { // helper function attempts to find and return the named secret. // from the secret provider. -func (c *Compiler) findSecret(ctx context.Context, args Args, name string) (s string, ok bool) { +func (c *Compiler) findSecret(ctx context.Context, args runtime.CompilerArgs, name string) (s string, ok bool) { if name == "" { return } diff --git a/engine/compiler/compiler_test.go b/engine/compiler/compiler_test.go index 621f40e..5c0b536 100644 --- a/engine/compiler/compiler_test.go +++ b/engine/compiler/compiler_test.go @@ -19,6 +19,7 @@ import ( "github.com/drone/drone-go/drone" "github.com/drone/runner-go/environ/provider" "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline/runtime" "github.com/drone/runner-go/registry" "github.com/drone/runner-go/secret" @@ -65,10 +66,10 @@ func TestCompile_CloneDisabled_Graph(t *testing.T) { // defined in the when block are not satisfied. func TestCompile_Match(t *testing.T) { ir := testCompile(t, "testdata/match.yml", "testdata/match.json") - if ir.Steps[0].RunPolicy != engine.RunOnSuccess { + if ir.Steps[0].RunPolicy != runtime.RunOnSuccess { t.Errorf("Expect run on success") } - if ir.Steps[1].RunPolicy != engine.RunNever { + if ir.Steps[1].RunPolicy != runtime.RunNever { t.Errorf("Expect run never") } } @@ -77,7 +78,7 @@ func TestCompile_Match(t *testing.T) { // success or failure are configured to always run. func TestCompile_RunAlways(t *testing.T) { ir := testCompile(t, "testdata/run_always.yml", "testdata/run_always.json") - if ir.Steps[0].RunPolicy != engine.RunAlways { + if ir.Steps[0].RunPolicy != runtime.RunAlways { t.Errorf("Expect run always") } } @@ -86,7 +87,7 @@ func TestCompile_RunAlways(t *testing.T) { // are configured to run on failure. func TestCompile_RunFailure(t *testing.T) { ir := testCompile(t, "testdata/run_failure.yml", "testdata/run_failure.json") - if ir.Steps[0].RunPolicy != engine.RunOnFailure { + if ir.Steps[0].RunPolicy != runtime.RunOnFailure { t.Errorf("Expect run on failure") } } diff --git a/engine/compiler/step.go b/engine/compiler/step.go index d0bd093..2b6213d 100644 --- a/engine/compiler/step.go +++ b/engine/compiler/step.go @@ -11,6 +11,8 @@ import ( "github.com/drone-runners/drone-runner-docker/engine/resource" "github.com/drone-runners/drone-runner-docker/internal/docker/image" "github.com/drone-runners/drone-runner-docker/internal/encoder" + + "github.com/drone/runner-go/pipeline/runtime" ) func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step { @@ -94,9 +96,9 @@ func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step { // success by default, but may be optionally configured // to run on failure. if isRunAlways(src) { - dst.RunPolicy = engine.RunAlways + dst.RunPolicy = runtime.RunAlways } else if isRunOnFailure(src) { - dst.RunPolicy = engine.RunOnFailure + dst.RunPolicy = runtime.RunOnFailure } return dst diff --git a/engine/const.go b/engine/const.go index 36ec1da..10878e5 100644 --- a/engine/const.go +++ b/engine/const.go @@ -61,58 +61,3 @@ func (p *PullPolicy) UnmarshalJSON(b []byte) error { *p = pullPolicyName[s] return nil } - -// RunPolicy defines the policy for starting containers -// based on the point-in-time pass or fail state of -// the pipeline. -type RunPolicy int - -// RunPolicy enumeration. -const ( - RunOnSuccess RunPolicy = iota - RunOnFailure - RunAlways - RunNever -) - -func (r RunPolicy) String() string { - return runPolicyID[r] -} - -var runPolicyID = map[RunPolicy]string{ - RunOnSuccess: "on-success", - RunOnFailure: "on-failure", - RunAlways: "always", - RunNever: "never", -} - -var runPolicyName = map[string]RunPolicy{ - "": RunOnSuccess, - "on-success": RunOnSuccess, - "on-failure": RunOnFailure, - "always": RunAlways, - "never": RunNever, -} - -// MarshalJSON marshals the string representation of the -// run type to JSON. -func (r *RunPolicy) MarshalJSON() ([]byte, error) { - buffer := bytes.NewBufferString(`"`) - buffer.WriteString(runPolicyID[*r]) - buffer.WriteString(`"`) - return buffer.Bytes(), nil -} - -// UnmarshalJSON unmarshals the json representation of the -// run type from a string value. -func (r *RunPolicy) UnmarshalJSON(b []byte) error { - // unmarshal as string - var s string - err := json.Unmarshal(b, &s) - if err != nil { - return err - } - // lookup value - *r = runPolicyName[s] - return nil -} diff --git a/engine/const_test.go b/engine/const_test.go index b671470..ef2bfc1 100644 --- a/engine/const_test.go +++ b/engine/const_test.go @@ -10,121 +10,6 @@ import ( "testing" ) -// -// runtime policy unit tests. -// - -func TestRunPolicy_Marshal(t *testing.T) { - tests := []struct { - policy RunPolicy - data string - }{ - { - policy: RunAlways, - data: `"always"`, - }, - { - policy: RunOnFailure, - data: `"on-failure"`, - }, - { - policy: RunOnSuccess, - data: `"on-success"`, - }, - { - policy: RunNever, - data: `"never"`, - }, - } - for _, test := range tests { - data, err := json.Marshal(&test.policy) - if err != nil { - t.Error(err) - return - } - if bytes.Equal([]byte(test.data), data) == false { - t.Errorf("Failed to marshal policy %s", test.policy) - } - } -} - -func TestRunPolicy_Unmarshal(t *testing.T) { - tests := []struct { - policy RunPolicy - data string - }{ - { - policy: RunAlways, - data: `"always"`, - }, - { - policy: RunOnFailure, - data: `"on-failure"`, - }, - { - policy: RunOnSuccess, - data: `"on-success"`, - }, - { - policy: RunNever, - data: `"never"`, - }, - { - // no policy should default to on-success - policy: RunOnSuccess, - data: `""`, - }, - } - for _, test := range tests { - var policy RunPolicy - err := json.Unmarshal([]byte(test.data), &policy) - if err != nil { - t.Error(err) - return - } - if got, want := policy, test.policy; got != want { - t.Errorf("Want policy %q, got %q", want, got) - } - } -} - -func TestRunPolicy_UnmarshalTypeError(t *testing.T) { - var policy RunPolicy - err := json.Unmarshal([]byte("[]"), &policy) - if _, ok := err.(*json.UnmarshalTypeError); !ok { - t.Errorf("Expect unmarshal error return when JSON invalid") - } -} - -func TestRunPolicy_String(t *testing.T) { - tests := []struct { - policy RunPolicy - value string - }{ - { - policy: RunAlways, - value: "always", - }, - { - policy: RunOnFailure, - value: "on-failure", - }, - { - policy: RunOnSuccess, - value: "on-success", - }, - } - for _, test := range tests { - if got, want := test.policy.String(), test.value; got != want { - t.Errorf("Want policy string %q, got %q", want, got) - } - } -} - -// -// pull policy unit tests. -// - func TestPullPolicy_Marshal(t *testing.T) { tests := []struct { policy PullPolicy diff --git a/engine/engine.go b/engine/engine.go index 02ec75d..b970aa8 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -7,17 +7,288 @@ package engine import ( "context" "io" + "io/ioutil" + + "github.com/drone-runners/drone-runner-docker/internal/docker/errors" + "github.com/drone-runners/drone-runner-docker/internal/docker/image" + "github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage" + "github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/registry/auths" + + "docker.io/go-docker" + "docker.io/go-docker/api/types" + "docker.io/go-docker/api/types/network" + "docker.io/go-docker/api/types/volume" ) -// Engine is the interface that must be implemented by a -// pipeline execution engine. -type Engine interface { - // Setup the pipeline environment. - Setup(context.Context, *Spec) error - - // Destroy the pipeline environment. - Destroy(context.Context, *Spec) error - - // Run runs the pipeine step. - Run(context.Context, *Spec, *Step, io.Writer) (*State, error) +// Opts configures the Docker engine. +type Opts struct { + HidePull bool +} + +// Docker implements a Docker pipeline engine. +type Docker struct { + client docker.APIClient + hidePull bool +} + +// New returns a new engine. +func New(client docker.APIClient, opts Opts) *Docker { + return &Docker{ + client: client, + hidePull: opts.HidePull, + } +} + +// NewEnv returns a new Engine from the environment. +func NewEnv(opts Opts) (*Docker, error) { + cli, err := docker.NewEnvClient() + if err != nil { + return nil, err + } + return New(cli, opts), nil +} + +// Ping pings the Docker daemon. +func (e *Docker) Ping(ctx context.Context) error { + _, err := e.client.Ping(ctx) + return err +} + +// Setup the pipeline environment. +func (e *Docker) Setup(ctx context.Context, specv runtime.Spec) error { + spec := specv.(*Spec) + + // creates the default temporary (local) volumes + // that are mounted into each container step. + for _, vol := range spec.Volumes { + if vol.EmptyDir == nil { + continue + } + _, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{ + Name: vol.EmptyDir.ID, + Driver: "local", + Labels: vol.EmptyDir.Labels, + }) + if err != nil { + return errors.TrimExtraInfo(err) + } + } + + // creates the default pod network. All containers + // defined in the pipeline are attached to this network. + driver := "bridge" + if spec.Platform.OS == "windows" { + driver = "nat" + } + _, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{ + Driver: driver, + Labels: spec.Network.Labels, + }) + + return errors.TrimExtraInfo(err) +} + +// Destroy the pipeline environment. +func (e *Docker) Destroy(ctx context.Context, specv runtime.Spec) error { + spec := specv.(*Spec) + + removeOpts := types.ContainerRemoveOptions{ + Force: true, + RemoveLinks: false, + RemoveVolumes: true, + } + + // stop all containers + for _, step := range spec.Steps { + e.client.ContainerKill(ctx, step.ID, "9") + } + + // cleanup all containers + for _, step := range spec.Steps { + e.client.ContainerRemove(ctx, step.ID, removeOpts) + } + + // cleanup all volumes + for _, vol := range spec.Volumes { + if vol.EmptyDir == nil { + continue + } + // tempfs volumes do not have a volume entry, + // and therefore do not require removal. + if vol.EmptyDir.Medium == "memory" { + continue + } + e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true) + } + + // cleanup the network + e.client.NetworkRemove(ctx, spec.Network.ID) + + // notice that we never collect or return any errors. + // this is because we silently ignore cleanup failures + // and instead ask the system admin to periodically run + // `docker prune` commands. + return nil +} + +// Run runs the pipeline step. +func (e *Docker) Run(ctx context.Context, specv runtime.Spec, stepv runtime.Step, output io.Writer) (*runtime.State, error) { + spec := specv.(*Spec) + step := stepv.(*Step) + + // create the container + err := e.create(ctx, spec, step, output) + if err != nil { + return nil, errors.TrimExtraInfo(err) + } + // start the container + err = e.start(ctx, step.ID) + if err != nil { + return nil, errors.TrimExtraInfo(err) + } + // tail the container + err = e.tail(ctx, step.ID, output) + if err != nil { + return nil, errors.TrimExtraInfo(err) + } + // wait for the response + return e.wait(ctx, step.ID) +} + +// +// emulate docker commands +// + +func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error { + // create pull options with encoded authorization credentials. + pullopts := types.ImagePullOptions{} + if step.Auth != nil { + pullopts.RegistryAuth = auths.Header( + step.Auth.Username, + step.Auth.Password, + ) + } + + // automatically pull the latest version of the image if requested + // by the process configuration, or if the image is :latest + if step.Pull == PullAlways || + (step.Pull == PullDefault && image.IsLatest(step.Image)) { + rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts) + if pullerr == nil { + if e.hidePull { + io.Copy(ioutil.Discard, rc) + } else { + jsonmessage.Copy(rc, output) + } + rc.Close() + } + if pullerr != nil { + return pullerr + } + } + + _, err := e.client.ContainerCreate(ctx, + toConfig(spec, step), + toHostConfig(spec, step), + toNetConfig(spec, step), + step.ID, + ) + + // automatically pull and try to re-create the image if the + // failure is caused because the image does not exist. + if docker.IsErrImageNotFound(err) && step.Pull != PullNever { + rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts) + if pullerr != nil { + return pullerr + } + + if e.hidePull { + io.Copy(ioutil.Discard, rc) + } else { + jsonmessage.Copy(rc, output) + } + rc.Close() + + // once the image is successfully pulled we attempt to + // re-create the container. + _, err = e.client.ContainerCreate(ctx, + toConfig(spec, step), + toHostConfig(spec, step), + toNetConfig(spec, step), + step.ID, + ) + } + if err != nil { + return err + } + + // attach the container to user-defined networks. + // primarily used to attach global user-defined networks. + if step.Network == "" { + for _, net := range step.Networks { + err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{ + Aliases: []string{net}, + }) + if err != nil { + return nil + } + } + } + + return nil +} + +// helper function emulates the `docker start` command. +func (e *Docker) start(ctx context.Context, id string) error { + return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{}) +} + +// helper function emulates the `docker wait` command, blocking +// until the container stops and returning the exit code. +func (e *Docker) wait(ctx context.Context, id string) (*runtime.State, error) { + wait, errc := e.client.ContainerWait(ctx, id, "") + select { + case <-wait: + case <-errc: + } + + info, err := e.client.ContainerInspect(ctx, id) + if err != nil { + return nil, err + } + if info.State.Running { + // TODO(bradrydewski) if the state is still running + // we should call wait again. + } + + return &runtime.State{ + Exited: true, + ExitCode: info.State.ExitCode, + OOMKilled: info.State.OOMKilled, + }, nil +} + +// helper function emulates the `docker logs -f` command, streaming +// all container logs until the container stops. +func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error { + opts := types.ContainerLogsOptions{ + Follow: true, + ShowStdout: true, + ShowStderr: true, + Details: false, + Timestamps: false, + } + + logs, err := e.client.ContainerLogs(ctx, id, opts) + if err != nil { + return err + } + + go func() { + stdcopy.StdCopy(output, output, logs) + logs.Close() + }() + return nil } diff --git a/engine/engine_impl.go b/engine/engine_impl.go deleted file mode 100644 index 1617a44..0000000 --- a/engine/engine_impl.go +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package engine - -import ( - "context" - "io" - "io/ioutil" - - "github.com/drone-runners/drone-runner-docker/internal/docker/errors" - "github.com/drone-runners/drone-runner-docker/internal/docker/image" - "github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage" - "github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy" - "github.com/drone/runner-go/registry/auths" - - "docker.io/go-docker" - "docker.io/go-docker/api/types" - "docker.io/go-docker/api/types/network" - "docker.io/go-docker/api/types/volume" -) - -// Opts configures the Docker engine. -type Opts struct { - HidePull bool -} - -// Docker implements a Docker pipeline engine. -type Docker struct { - client docker.APIClient - hidePull bool -} - -// New returns a new engine. -func New(client docker.APIClient, opts Opts) *Docker { - return &Docker{ - client: client, - hidePull: opts.HidePull, - } -} - -// NewEnv returns a new Engine from the environment. -func NewEnv(opts Opts) (*Docker, error) { - cli, err := docker.NewEnvClient() - if err != nil { - return nil, err - } - return New(cli, opts), nil -} - -// Ping pings the Docker daemon. -func (e *Docker) Ping(ctx context.Context) error { - _, err := e.client.Ping(ctx) - return err -} - -// Setup the pipeline environment. -func (e *Docker) Setup(ctx context.Context, spec *Spec) error { - // creates the default temporary (local) volumes - // that are mounted into each container step. - for _, vol := range spec.Volumes { - if vol.EmptyDir == nil { - continue - } - _, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{ - Name: vol.EmptyDir.ID, - Driver: "local", - Labels: vol.EmptyDir.Labels, - }) - if err != nil { - return errors.TrimExtraInfo(err) - } - } - - // creates the default pod network. All containers - // defined in the pipeline are attached to this network. - driver := "bridge" - if spec.Platform.OS == "windows" { - driver = "nat" - } - _, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{ - Driver: driver, - Labels: spec.Network.Labels, - }) - - return errors.TrimExtraInfo(err) -} - -// Destroy the pipeline environment. -func (e *Docker) Destroy(ctx context.Context, spec *Spec) error { - removeOpts := types.ContainerRemoveOptions{ - Force: true, - RemoveLinks: false, - RemoveVolumes: true, - } - - // stop all containers - for _, step := range spec.Steps { - e.client.ContainerKill(ctx, step.ID, "9") - } - - // cleanup all containers - for _, step := range spec.Steps { - e.client.ContainerRemove(ctx, step.ID, removeOpts) - } - - // cleanup all volumes - for _, vol := range spec.Volumes { - if vol.EmptyDir == nil { - continue - } - // tempfs volumes do not have a volume entry, - // and therefore do not require removal. - if vol.EmptyDir.Medium == "memory" { - continue - } - e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true) - } - - // cleanup the network - e.client.NetworkRemove(ctx, spec.Network.ID) - - // notice that we never collect or return any errors. - // this is because we silently ignore cleanup failures - // and instead ask the system admin to periodically run - // `docker prune` commands. - return nil -} - -// Run runs the pipeline step. -func (e *Docker) Run(ctx context.Context, spec *Spec, step *Step, output io.Writer) (*State, error) { - // create the container - err := e.create(ctx, spec, step, output) - if err != nil { - return nil, errors.TrimExtraInfo(err) - } - // start the container - err = e.start(ctx, step.ID) - if err != nil { - return nil, errors.TrimExtraInfo(err) - } - // tail the container - err = e.tail(ctx, step.ID, output) - if err != nil { - return nil, errors.TrimExtraInfo(err) - } - // wait for the response - return e.wait(ctx, step.ID) -} - -// -// emulate docker commands -// - -func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error { - // create pull options with encoded authorization credentials. - pullopts := types.ImagePullOptions{} - if step.Auth != nil { - pullopts.RegistryAuth = auths.Header( - step.Auth.Username, - step.Auth.Password, - ) - } - - // automatically pull the latest version of the image if requested - // by the process configuration, or if the image is :latest - if step.Pull == PullAlways || - (step.Pull == PullDefault && image.IsLatest(step.Image)) { - rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts) - if pullerr == nil { - if e.hidePull { - io.Copy(ioutil.Discard, rc) - } else { - jsonmessage.Copy(rc, output) - } - rc.Close() - } - if pullerr != nil { - return pullerr - } - } - - _, err := e.client.ContainerCreate(ctx, - toConfig(spec, step), - toHostConfig(spec, step), - toNetConfig(spec, step), - step.ID, - ) - - // automatically pull and try to re-create the image if the - // failure is caused because the image does not exist. - if docker.IsErrImageNotFound(err) && step.Pull != PullNever { - rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts) - if pullerr != nil { - return pullerr - } - - if e.hidePull { - io.Copy(ioutil.Discard, rc) - } else { - jsonmessage.Copy(rc, output) - } - rc.Close() - - // once the image is successfully pulled we attempt to - // re-create the container. - _, err = e.client.ContainerCreate(ctx, - toConfig(spec, step), - toHostConfig(spec, step), - toNetConfig(spec, step), - step.ID, - ) - } - if err != nil { - return err - } - - // attach the container to user-defined networks. - // primarily used to attach global user-defined networks. - if step.Network == "" { - for _, net := range step.Networks { - err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{ - Aliases: []string{net}, - }) - if err != nil { - return nil - } - } - } - - return nil -} - -// helper function emulates the `docker start` command. -func (e *Docker) start(ctx context.Context, id string) error { - return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{}) -} - -// helper function emulates the `docker wait` command, blocking -// until the container stops and returning the exit code. -func (e *Docker) wait(ctx context.Context, id string) (*State, error) { - wait, errc := e.client.ContainerWait(ctx, id, "") - select { - case <-wait: - case <-errc: - } - - info, err := e.client.ContainerInspect(ctx, id) - if err != nil { - return nil, err - } - if info.State.Running { - // TODO(bradrydewski) if the state is still running - // we should call wait again. - } - - return &State{ - Exited: true, - ExitCode: info.State.ExitCode, - OOMKilled: info.State.OOMKilled, - }, nil -} - -// helper function emulates the `docker logs -f` command, streaming -// all container logs until the container stops. -func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error { - opts := types.ContainerLogsOptions{ - Follow: true, - ShowStdout: true, - ShowStderr: true, - Details: false, - Timestamps: false, - } - - logs, err := e.client.ContainerLogs(ctx, id, opts) - if err != nil { - return err - } - - go func() { - stdcopy.StdCopy(output, output, logs) - logs.Close() - }() - return nil -} diff --git a/engine/linter/linter.go b/engine/linter/linter.go index bf43c4f..58921a7 100644 --- a/engine/linter/linter.go +++ b/engine/linter/linter.go @@ -11,6 +11,8 @@ import ( "strings" "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/manifest" ) // ErrDuplicateStepName is returned when two Pipeline steps @@ -43,8 +45,8 @@ func New() *Linter { // Lint executes the linting rules for the pipeline // configuration. -func (l *Linter) Lint(pipeline *resource.Pipeline, opts Opts) error { - return checkPipeline(pipeline, opts.Trusted) +func (l *Linter) Lint(pipeline manifest.Resource, repo *drone.Repo) error { + return checkPipeline(pipeline.(*resource.Pipeline), repo.Trusted) } func checkPipeline(pipeline *resource.Pipeline, trusted bool) error { diff --git a/engine/linter/linter_test.go b/engine/linter/linter_test.go index ab51e33..ac58537 100644 --- a/engine/linter/linter_test.go +++ b/engine/linter/linter_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" "github.com/drone/runner-go/manifest" ) @@ -212,7 +213,7 @@ func TestLint(t *testing.T) { } lint := New() - opts := Opts{Trusted: test.trusted} + opts := &drone.Repo{Trusted: test.trusted} err = lint.Lint(resources.Resources[0].(*resource.Pipeline), opts) if err == nil && test.invalid == true { t.Logf("yaml: %s", test.path) diff --git a/engine/replacer/replacer.go b/engine/replacer/replacer.go deleted file mode 100644 index 3db9844..0000000 --- a/engine/replacer/replacer.go +++ /dev/null @@ -1,56 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package replacer - -import ( - "fmt" - "io" - "strings" - - "github.com/drone-runners/drone-runner-docker/engine" -) - -const maskedf = "[secret:%s]" - -// Replacer is an io.Writer that finds and masks sensitive data. -type Replacer struct { - w io.WriteCloser - r *strings.Replacer -} - -// New returns a replacer that wraps writer w. -func New(w io.WriteCloser, secrets []*engine.Secret) io.WriteCloser { - var oldnew []string - for _, secret := range secrets { - if len(secret.Data) == 0 || secret.Mask == false { - continue - } - name := strings.ToLower(secret.Name) - masked := fmt.Sprintf(maskedf, name) - oldnew = append(oldnew, string(secret.Data)) - oldnew = append(oldnew, masked) - } - if len(oldnew) == 0 { - return w - } - return &Replacer{ - w: w, - r: strings.NewReplacer(oldnew...), - } -} - -// Write writes p to the base writer. The method scans for any -// sensitive data in p and masks before writing. -func (r *Replacer) Write(p []byte) (n int, err error) { - _, err = r.w.Write([]byte(r.r.Replace(string(p)))) - return len(p), err -} - -// Close closes the base writer. -func (r *Replacer) Close() error { - return r.w.Close() -} diff --git a/engine/replacer/replacer_test.go b/engine/replacer/replacer_test.go deleted file mode 100644 index 54c7462..0000000 --- a/engine/replacer/replacer_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package replacer - -import ( - "bytes" - "io" - "testing" - - "github.com/drone-runners/drone-runner-docker/engine" -) - -func TestReplace(t *testing.T) { - secrets := []*engine.Secret{ - {Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false}, - {Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: true}, - {Name: "DOCKER_EMAIL", Data: []byte(""), Mask: true}, - } - - buf := new(bytes.Buffer) - w := New(&nopCloser{buf}, secrets) - w.Write([]byte("username octocat password correct-horse-batter-staple")) - w.Close() - - if got, want := buf.String(), "username octocat password [secret:docker_password]"; got != want { - t.Errorf("Want masked string %s, got %s", want, got) - } -} - -// this test verifies that if there are no secrets to scan and -// mask, the io.WriteCloser is returned as-is. -func TestReplaceNone(t *testing.T) { - secrets := []*engine.Secret{ - {Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false}, - {Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: false}, - } - - buf := new(bytes.Buffer) - w := &nopCloser{buf} - r := New(w, secrets) - if w != r { - t.Errorf("Expect buffer returned with no replacer") - } -} - -type nopCloser struct { - io.Writer -} - -func (*nopCloser) Close() error { - return nil -} diff --git a/engine/resource/lookup.go b/engine/resource/lookup.go index 4db0113..17a75bc 100644 --- a/engine/resource/lookup.go +++ b/engine/resource/lookup.go @@ -11,7 +11,7 @@ import ( ) // Lookup returns the named pipeline from the Manifest. -func Lookup(name string, manifest *manifest.Manifest) (*Pipeline, error) { +func Lookup(name string, manifest *manifest.Manifest) (manifest.Resource, error) { for _, resource := range manifest.Resources { if !isNameMatch(resource.GetName(), name) { continue diff --git a/engine/spec.go b/engine/spec.go index 9b524dc..fb8b08f 100644 --- a/engine/spec.go +++ b/engine/spec.go @@ -4,7 +4,13 @@ package engine +import ( + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/pipeline/runtime" +) + type ( + // Spec provides the pipeline spec. This provides the // required instructions for reproducible pipeline // execution. @@ -44,7 +50,7 @@ type ( Networks []string `json:"networks,omitempty"` Privileged bool `json:"privileged,omitempty"` Pull PullPolicy `json:"pull,omitempty"` - RunPolicy RunPolicy `json:"run_policy,omitempty"` + RunPolicy runtime.RunPolicy `json:"run_policy,omitempty"` Secrets []*Secret `json:"secrets,omitempty"` ShmSize int64 `json:"shm_size,omitempty"` User string `json:"user,omitempty"` @@ -52,14 +58,6 @@ type ( WorkingDir string `json:"working_dir,omitempty"` } - // Platform defines the target platform. - Platform struct { - OS string `json:"os,omitempty"` - Arch string `json:"arch,omitempty"` - Variant string `json:"variant,omitempty"` - Version string `json:"version,omitempty"` - } - // Secret represents a secret variable. Secret struct { Name string `json:"name,omitempty"` @@ -68,11 +66,12 @@ type ( Mask bool `json:"mask,omitempty"` } - // State represents the process state. - State struct { - ExitCode int // Container exit code - Exited bool // Container exited - OOMKilled bool // Container is oom killed + // Platform defines the target platform. + Platform struct { + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + Variant string `json:"variant,omitempty"` + Version string `json:"version,omitempty"` } // Volume that can be mounted by containers. @@ -128,3 +127,38 @@ type ( Password string `json:"password,omitempty"` } ) + +// +// implements the Spec interface +// + +func (s *Spec) StepLen() int { return len(s.Steps) } +func (s *Spec) StepAt(i int) runtime.Step { return s.Steps[i] } + +// +// implements the Secret interface +// + +func (s *Secret) GetName() string { return s.Name } +func (s *Secret) GetValue() string { return string(s.Data) } +func (s *Secret) IsMasked() bool { return s.Mask } + +// +// implements the Step interface +// + +func (s *Step) GetName() string { return s.Name } +func (s *Step) GetDependencies() []string { return s.DependsOn } +func (s *Step) GetEnviron() map[string]string { return s.Envs } +func (s *Step) SetEnviron(env map[string]string) { s.Envs = env } +func (s *Step) GetErrPolicy() runtime.ErrPolicy { return runtime.ErrFail } +func (s *Step) GetRunPolicy() runtime.RunPolicy { return s.RunPolicy } +func (s *Step) GetSecretAt(i int) runtime.Secret { return s.Secrets[i] } +func (s *Step) GetSecretLen() int { return len(s.Secrets) } +func (s *Step) IsDetached() bool { return s.Detach } +func (s *Step) Clone() runtime.Step { + dst := new(Step) + *dst = *s + dst.Envs = environ.Combine(s.Envs) + return dst +} diff --git a/go.mod b/go.mod index 3d38db0..2ef1234 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,17 @@ go 1.12 require ( docker.io/go-docker v1.0.0 - github.com/Microsoft/go-winio v0.4.11 // indirect github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect github.com/buildkite/yaml v2.1.0+incompatible github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 github.com/docker/distribution v2.7.1+incompatible - github.com/docker/go-connections v0.3.0 // indirect github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79 + github.com/drone/drone-runtime v1.1.0 github.com/drone/envsubst v1.0.2 - github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7 + github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c github.com/drone/signal v1.0.0 github.com/ghodss/yaml v1.0.0 - github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 // indirect github.com/golang/mock v1.3.1 github.com/google/go-cmp v0.3.0 github.com/hashicorp/go-multierror v1.0.0 @@ -25,8 +23,6 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-isatty v0.0.8 github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pkg/errors v0.8.1 // indirect github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.3.0 // indirect diff --git a/go.sum b/go.sum index 6dc5a84..8882ab5 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o= github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba h1:GKiT4UPBligLXJAP1zRllHvTUygAAlgS3t9LM9aasp0= @@ -32,6 +34,8 @@ github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba/go.mod h1:GxyeGCl github.com/drone/drone-go v1.1.0 h1:2mritc5b7PhQWvILNyzaImZMRWVbMmmZ5Q0UDwwO7SI= github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79 h1:jW+dJ8HrZ1CbazlsYoriOOCQnVJ2NkfNczLHs6UMU6I= github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4= +github.com/drone/drone-runtime v1.1.0 h1:IsKbwiLY6+ViNBzX0F8PERJVZZcEJm9rgxEh3uZP5IE= +github.com/drone/drone-runtime v1.1.0/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs= github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo= github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0= github.com/drone/runner-go v1.2.2 h1:fwYgjyJl6KdjQGEUFof9+HLtNpK3iHq7UuR+/aYNyDk= @@ -46,23 +50,39 @@ github.com/drone/runner-go v1.3.0/go.mod h1:61VgQWhZbNPXp01lBuR7PAztTMySGLnMzK/4 github.com/drone/runner-go v1.4.0 h1:zAeYtlKQGvJr2ehfLzQWblzWzvfdkNaapbr6x536fLA= github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7 h1:iNLp8xT0rMcV/tT2J3fCuEbWLqoP7CiL3WR8W3i3HpQ= github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7/go.mod h1:IqwuMbIoeH45k4NemcNPwymm+l386EeyBC166UElURw= +github.com/drone/runner-go v1.4.1-0.20191206074106-b546cbadce98 h1:281h7RjcgHLOsqP0abNrS8ESVhtNks2i/WWW0+S0HWw= +github.com/drone/runner-go v1.4.1-0.20191206074106-b546cbadce98/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o= +github.com/drone/runner-go v1.4.1-0.20191206204642-f16a481b8b8f h1:TC1w5LWTOEbbatXnx92vRqsEsiIenzwWMmIqEvYqo+U= +github.com/drone/runner-go v1.4.1-0.20191206204642-f16a481b8b8f/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o= +github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c h1:T3CIdRKiTiaNbInLmtPOoY/9BNUPaL2ipBr2jHzaJsE= +github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o= github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI= github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 h1:zDlw+wgyXdfkRuvFCdEDUiPLmZp2cvf/dWHazY0a5VM= github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -72,33 +92,47 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 h1:dnMxwus89s86tI8rcGVp2HwZzlz7c5o92VOy7dSckBQ= github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4/go.mod h1:cojhOHk1gbMeklOyDP2oKKLftefXoJreOQGOrXk+Z38= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -106,11 +140,19 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/runtime/execer.go b/runtime/execer.go deleted file mode 100644 index 9071abf..0000000 --- a/runtime/execer.go +++ /dev/null @@ -1,242 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime - -import ( - "context" - "sync" - - "github.com/drone-runners/drone-runner-docker/engine" - "github.com/drone-runners/drone-runner-docker/engine/replacer" - "github.com/drone/drone-go/drone" - "github.com/drone/runner-go/environ" - "github.com/drone/runner-go/logger" - "github.com/drone/runner-go/pipeline" - - "github.com/hashicorp/go-multierror" - "github.com/natessilva/dag" - "golang.org/x/sync/semaphore" -) - -// Execer is the execution context for executing the intermediate -// representation of a pipeline. -type Execer interface { - Exec(context.Context, *engine.Spec, *pipeline.State) error -} - -type execer struct { - mu sync.Mutex - engine engine.Engine - reporter pipeline.Reporter - streamer pipeline.Streamer - sem *semaphore.Weighted -} - -// NewExecer returns a new execer used -func NewExecer( - reporter pipeline.Reporter, - streamer pipeline.Streamer, - engine engine.Engine, - procs int64, -) Execer { - exec := &execer{ - reporter: reporter, - streamer: streamer, - engine: engine, - } - if procs > 0 { - // optional semaphor that limits the number of steps - // that can execute concurrently. - exec.sem = semaphore.NewWeighted(procs) - } - return exec -} - -// Exec executes the intermediate representation of the pipeline -// and returns an error if execution fails. -func (e *execer) Exec(ctx context.Context, spec *engine.Spec, state *pipeline.State) error { - defer e.engine.Destroy(noContext, spec) - - if err := e.engine.Setup(noContext, spec); err != nil { - state.FailAll(err) - return e.reporter.ReportStage(noContext, state) - } - - // create a directed graph, where each vertex in the graph - // is a pipeline step. - var d dag.Runner - for _, s := range spec.Steps { - step := s - d.AddVertex(step.Name, func() error { - return e.exec(ctx, state, spec, step) - }) - } - - // create the vertex edges from the values configured in the - // depends_on attribute. - for _, s := range spec.Steps { - for _, dep := range s.DependsOn { - d.AddEdge(dep, s.Name) - } - } - - var result error - if err := d.Run(); err != nil { - multierror.Append(result, err) - } - - // once pipeline execution completes, notify the state - // manageer that all steps are finished. - state.FinishAll() - if err := e.reporter.ReportStage(noContext, state); err != nil { - multierror.Append(result, err) - } - return result -} - -func (e *execer) exec(ctx context.Context, state *pipeline.State, spec *engine.Spec, step *engine.Step) error { - var result error - - select { - case <-ctx.Done(): - state.Cancel() - return nil - default: - } - - log := logger.FromContext(ctx) - log = log.WithField("step.name", step.Name) - ctx = logger.WithContext(ctx, log) - - if e.sem != nil { - // the semaphore limits the number of steps that can run - // concurrently. acquire the semaphore and release when - // the pipeline completes. - if err := e.sem.Acquire(ctx, 1); err != nil { - return nil - } - - defer func() { - // recover from a panic to ensure the semaphore is - // released to prevent deadlock. we do not expect a - // panic, however, we are being overly cautious. - if r := recover(); r != nil { - // TODO(bradrydzewsi) log the panic. - } - // release the semaphore - e.sem.Release(1) - }() - } - - switch { - case state.Skipped(): - return nil - case state.Cancelled(): - return nil - case step.RunPolicy == engine.RunNever: - return nil - case step.RunPolicy == engine.RunAlways: - break - case step.RunPolicy == engine.RunOnFailure && state.Failed() == false: - state.Skip(step.Name) - return e.reporter.ReportStep(noContext, state, step.Name) - case step.RunPolicy == engine.RunOnSuccess && state.Failed(): - state.Skip(step.Name) - return e.reporter.ReportStep(noContext, state, step.Name) - } - - state.Start(step.Name) - err := e.reporter.ReportStep(noContext, state, step.Name) - if err != nil { - return err - } - - copy := cloneStep(step) - - // the pipeline environment variables need to be updated to - // reflect the current state of the build and stage. - state.Lock() - copy.Envs = environ.Combine( - copy.Envs, - environ.Build(state.Build), - environ.Stage(state.Stage), - environ.Step(findStep(state, step.Name)), - ) - state.Unlock() - - // writer used to stream build logs. - wc := e.streamer.Stream(noContext, state, step.Name) - wc = replacer.New(wc, step.Secrets) - - // if the step is configured as a daemon, it is detached - // from the main process and executed separately. - // todo(bradrydzewski) this code is still experimental. - if step.Detach { - go func() { - e.engine.Run(ctx, spec, copy, wc) - wc.Close() - }() - return nil - } - - exited, err := e.engine.Run(ctx, spec, copy, wc) - - // close the stream. If the session is a remote session, the - // full log buffer is uploaded to the remote server. - if err := wc.Close(); err != nil { - multierror.Append(result, err) - } - - if exited != nil { - state.Finish(step.Name, exited.ExitCode) - err := e.reporter.ReportStep(noContext, state, step.Name) - if err != nil { - multierror.Append(result, err) - } - // if the exit code is 78 the system will skip all - // subsequent pending steps in the pipeline. - if exited.ExitCode == 78 { - state.SkipAll() - } - return result - } - - switch err { - case context.Canceled, context.DeadlineExceeded: - state.Cancel() - return nil - } - - // if the step failed with an internal error (as oppsed to a - // runtime error) the step is failed. - state.Fail(step.Name, err) - err = e.reporter.ReportStep(noContext, state, step.Name) - if err != nil { - multierror.Append(result, err) - } - return result -} - -// helper function to clone a step. The runner mutates a step to -// update the environment variables to reflect the current -// pipeline state. -func cloneStep(src *engine.Step) *engine.Step { - dst := new(engine.Step) - *dst = *src - dst.Envs = environ.Combine(src.Envs) - return dst -} - -// helper function returns the named step from the state. -func findStep(state *pipeline.State, name string) *drone.Step { - for _, step := range state.Stage.Steps { - if step.Name == name { - return step - } - } - panic("step not found: " + name) -} diff --git a/runtime/execer_test.go b/runtime/execer_test.go deleted file mode 100644 index 784ab5c..0000000 --- a/runtime/execer_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime - -import ( - "testing" -) - -func TestExec(t *testing.T) { - t.Skip() -} - -func TestExec_NonZeroExit(t *testing.T) { - t.Skip() -} - -func TestExec_Exit78(t *testing.T) { - t.Skip() -} - -func TestExec_Error(t *testing.T) { - t.Skip() -} - -func TestExec_CtxError(t *testing.T) { - t.Skip() -} - -func TestExec_ReportError(t *testing.T) { - t.Skip() -} - -func TestExec_SkipCtxDone(t *testing.T) { - t.Skip() -} diff --git a/runtime/poller.go b/runtime/poller.go deleted file mode 100644 index f7908e6..0000000 --- a/runtime/poller.go +++ /dev/null @@ -1,76 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime - -import ( - "context" - "sync" - - "github.com/drone/runner-go/client" - "github.com/drone/runner-go/logger" -) - -var noContext = context.Background() - -// Poller polls the server for pending stages and dispatches -// for execution by the Runner. -type Poller struct { - Client client.Client - Filter *client.Filter - Runner *Runner -} - -// Poll opens N connections to the server to poll for pending -// stages for execution. Pending stages are dispatched to a -// Runner for execution. -func (p *Poller) Poll(ctx context.Context, n int) { - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - for { - select { - case <-ctx.Done(): - wg.Done() - return - default: - p.poll(ctx, i+1) - } - } - }(i) - } - - wg.Wait() -} - -// poll requests a stage for execution from the server, and then -// dispatches for execution. -func (p *Poller) poll(ctx context.Context, thread int) error { - log := logger.FromContext(ctx).WithField("thread", thread) - log.WithField("thread", thread).Debug("request stage from remote server") - - // request a new build stage for execution from the central - // build server. - stage, err := p.Client.Request(ctx, p.Filter) - if err == context.Canceled || err == context.DeadlineExceeded { - log.WithError(err).Trace("no stage returned") - return nil - } - if err != nil { - log.WithError(err).Error("cannot request stage") - return err - } - - // exit if a nil or empty stage is returned from the system - // and allow the runner to retry. - if stage == nil || stage.ID == 0 { - return nil - } - - return p.Runner.Run( - logger.WithContext(noContext, log), stage) -} diff --git a/runtime/poller_test.go b/runtime/poller_test.go deleted file mode 100644 index 519a535..0000000 --- a/runtime/poller_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime - -import ( - "testing" -) - -func TestPoll(t *testing.T) { - t.Skip() -} - -func TestPoll_NilStage(t *testing.T) { - t.Skip() -} - -func TestPoll_EmptyStage(t *testing.T) { - t.Skip() -} - -func TestPoll_RequestError(t *testing.T) { - t.Skip() -} diff --git a/runtime/runner.go b/runtime/runner.go deleted file mode 100644 index 96a70f1..0000000 --- a/runtime/runner.go +++ /dev/null @@ -1,249 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/drone-runners/drone-runner-docker/engine" - "github.com/drone-runners/drone-runner-docker/engine/compiler" - "github.com/drone-runners/drone-runner-docker/engine/linter" - "github.com/drone-runners/drone-runner-docker/engine/resource" - - "github.com/drone/drone-go/drone" - "github.com/drone/envsubst" - "github.com/drone/runner-go/client" - "github.com/drone/runner-go/environ" - "github.com/drone/runner-go/logger" - "github.com/drone/runner-go/manifest" - "github.com/drone/runner-go/pipeline" - "github.com/drone/runner-go/secret" -) - -// Runner runs the pipeline. -type Runner struct { - // Client is the remote client responsible for interacting - // with the central server. - Client client.Client - - // Compiler is responsible for compiling the pipeline - // configuration to the intermediate representation. - Compiler *compiler.Compiler - - // Execer is responsible for executing intermediate - // representation of the pipeline and returns its results. - Execer Execer - - // Linter is responsible for linting the pipeline - // and failing if any rules are broken. - Linter *linter.Linter - - // Machine provides the runner with the name of the host - // machine executing the pipeline. - Machine string - - // Match is an optional function that returns true if the - // repository or build match user-defined criteria. This is - // intended as a security measure to prevent a runner from - // processing an unwanted pipeline. - Match func(*drone.Repo, *drone.Build) bool - - // Reporter reports pipeline status and logs back to the - // remote server. - Reporter pipeline.Reporter -} - -// Run runs the pipeline stage. -func (s *Runner) Run(ctx context.Context, stage *drone.Stage) error { - log := logger.FromContext(ctx). - WithField("stage.id", stage.ID). - WithField("stage.name", stage.Name). - WithField("stage.number", stage.Number) - - log.Debug("stage received") - - // delivery to a single agent is not guaranteed, which means - // we need confirm receipt. The first agent that confirms - // receipt of the stage can assume ownership. - - stage.Machine = s.Machine - err := s.Client.Accept(ctx, stage) - if err != nil && err == client.ErrOptimisticLock { - log.Debug("stage accepted by another runner") - return nil - } - if err != nil { - log.WithError(err).Error("cannot accept stage") - return err - } - - log.Debug("stage accepted") - - data, err := s.Client.Detail(ctx, stage) - if err != nil { - log.WithError(err).Error("cannot get stage details") - return err - } - - log = log.WithField("repo.id", data.Repo.ID). - WithField("repo.namespace", data.Repo.Namespace). - WithField("repo.name", data.Repo.Name). - WithField("build.id", data.Build.ID). - WithField("build.number", data.Build.Number) - - log.Debug("stage details fetched") - - ctxdone, cancel := context.WithCancel(ctx) - defer cancel() - - timeout := time.Duration(data.Repo.Timeout) * time.Minute - ctxtimeout, cancel := context.WithTimeout(ctxdone, timeout) - defer cancel() - - ctxcancel, cancel := context.WithCancel(ctxtimeout) - defer cancel() - - // next we opens a connection to the server to watch for - // cancellation requests. If a build is cancelled the running - // stage should also be cancelled. - go func() { - done, _ := s.Client.Watch(ctxdone, data.Build.ID) - if done { - cancel() - log.Debugln("received cancellation") - } else { - log.Debugln("done listening for cancellations") - } - }() - - envs := environ.Combine( - environ.System(data.System), - environ.Repo(data.Repo), - environ.Build(data.Build), - environ.Stage(stage), - environ.Link(data.Repo, data.Build, data.System), - data.Build.Params, - ) - - // string substitution function ensures that string - // replacement variables are escaped and quoted if they - // contain a newline character. - subf := func(k string) string { - v := envs[k] - if strings.Contains(v, "\n") { - v = fmt.Sprintf("%q", v) - } - return v - } - - state := &pipeline.State{ - Build: data.Build, - Stage: stage, - Repo: data.Repo, - System: data.System, - } - - // evaluates whether or not the agent can process the - // pipeline. An agent may choose to reject a repository - // or build for security reasons. - if s.Match != nil && s.Match(data.Repo, data.Build) == false { - log.Error("cannot process stage, access denied") - state.FailAll(errors.New("insufficient permission to run the pipeline")) - return s.Reporter.ReportStage(noContext, state) - } - - // evaluates string replacement expressions and returns an - // update configuration file string. - config, err := envsubst.Eval(string(data.Config.Data), subf) - if err != nil { - log.WithError(err).Error("cannot emulate bash substitution") - state.FailAll(err) - return s.Reporter.ReportStage(noContext, state) - } - - // parse the yaml configuration file. - manifest, err := manifest.ParseString(config) - if err != nil { - log.WithError(err).Error("cannot parse configuration file") - state.FailAll(err) - return s.Reporter.ReportStage(noContext, state) - } - - // find the named stage in the yaml configuration file. - resource, err := resource.Lookup(stage.Name, manifest) - if err != nil { - log.WithError(err).Error("cannot find pipeline resource") - state.FailAll(err) - return s.Reporter.ReportStage(noContext, state) - } - - // lint the pipeline configuration and fail the build - // if any linting rules are broken. - err = s.Linter.Lint(resource, linter.Opts{Trusted: data.Repo.Trusted}) - if err != nil { - log.WithError(err).Error("cannot accept configuration") - state.FailAll(err) - return s.Reporter.ReportStage(noContext, state) - } - - secrets := secret.Combine( - secret.Static(data.Secrets), - secret.Encrypted(), - // s.Secret, - ) - - // compile the yaml configuration file to an intermediate - // representation, and then - args := compiler.Args{ - Pipeline: resource, - Manifest: manifest, - Build: data.Build, - Stage: stage, - Repo: data.Repo, - System: data.System, - Netrc: data.Netrc, - Secret: secrets, - } - - spec := s.Compiler.Compile(ctx, args) - for _, src := range spec.Steps { - // steps that are skipped are ignored and are not stored - // in the drone database, nor displayed in the UI. - if src.RunPolicy == engine.RunNever { - continue - } - stage.Steps = append(stage.Steps, &drone.Step{ - Name: src.Name, - Number: len(stage.Steps) + 1, - StageID: stage.ID, - Status: drone.StatusPending, - ErrIgnore: src.IgnoreErr, - }) - } - - stage.Started = time.Now().Unix() - stage.Status = drone.StatusRunning - if err := s.Client.Update(ctx, stage); err != nil { - log.WithError(err).Error("cannot update stage") - return err - } - - log.Debug("updated stage to running") - - ctxcancel = logger.WithContext(ctxcancel, log) - err = s.Execer.Exec(ctxcancel, spec, state) - if err != nil { - log.WithError(err).Debug("stage failed") - return err - } - log.Debug("updated stage to complete") - return nil -} diff --git a/runtime/runner_test.go b/runtime/runner_test.go deleted file mode 100644 index 69aede2..0000000 --- a/runtime/runner_test.go +++ /dev/null @@ -1,7 +0,0 @@ -// Code generated automatically. DO NOT EDIT. - -// Copyright 2019 Drone.IO Inc. All rights reserved. -// Use of this source code is governed by the Polyform License -// that can be found in the LICENSE file. - -package runtime