chaos/drone-runner-podman
Archived
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

abstract polling and execution to runner-go library

Browse Source
This commit is contained in:
Brad Rydzewski
2019-12-06 16:10:27 -08:00
parent 99e80a0352
commit 2e48cd1b3b
27 changed files with 500 additions and 1394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -4,6 +4,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Changed
- abstract polling and execution to runner-go library
- use trace level logging for context errors
- prefix docker resource names

6
command/compile.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
@@ -90,8 +91,7 @@ func (c *compileCommand) run(*kingpin.ParseContext) error {
// lint the pipeline and return an error if any
// linting rules are broken
lint := linter.New()
opts := linter.Opts{Trusted: c.Repo.Trusted}
err = lint.Lint(resource, opts)
err = lint.Lint(resource, c.Repo)
if err != nil {
return err
}
@@ -117,7 +117,7 @@ func (c *compileCommand) run(*kingpin.ParseContext) error {
comp.Mount, _ = os.Getwd()
}
args := compiler.Args{
args := runtime.CompilerArgs{
Pipeline: resource,
Manifest: manifest,
Build: c.Build,

126
command/daemon/daemon.go
View File

@@ -13,15 +13,16 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/linter"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/match"
"github.com/drone-runners/drone-runner-docker/runtime"
"github.com/drone/runner-go/client"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/handler/router"
"github.com/drone/runner-go/logger"
loghistory "github.com/drone/runner-go/logger/history"
"github.com/drone/runner-go/pipeline/history"
"github.com/drone/runner-go/pipeline/remote"
"github.com/drone/runner-go/pipeline/reporter/history"
"github.com/drone/runner-go/pipeline/reporter/remote"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/poller"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
"github.com/drone/runner-go/server"
@@ -107,67 +108,70 @@ func (c *daemonCommand) run(*kingpin.ParseContext) error {
hook := loghistory.New()
logrus.AddHook(hook)
poller := &runtime.Poller{
Client: cli,
Runner: &runtime.Runner{
Client: cli,
Machine: config.Runner.Name,
Reporter: tracer,
Linter: linter.New(),
Match: match.Func(
config.Limit.Repos,
config.Limit.Events,
config.Limit.Trusted,
),
Compiler: &compiler.Compiler{
Clone: config.Runner.Clone,
Privileged: append(config.Runner.Privileged, compiler.Privileged...),
Networks: config.Runner.Networks,
Volumes: config.Runner.Volumes,
Resources: compiler.Resources{
Memory: config.Resources.Memory,
MemorySwap: config.Resources.MemorySwap,
CPUQuota: config.Resources.CPUQuota,
CPUPeriod: config.Resources.CPUPeriod,
CPUShares: config.Resources.CPUShares,
CPUSet: config.Resources.CPUSet,
},
Environ: provider.Combine(
provider.Static(config.Runner.Environ),
provider.External(
config.Environ.Endpoint,
config.Environ.Token,
config.Environ.SkipVerify,
),
),
Registry: registry.Combine(
registry.File(
config.Docker.Config,
),
registry.External(
config.Registry.Endpoint,
config.Registry.Token,
config.Registry.SkipVerify,
),
),
Secret: secret.Combine(
secret.StaticVars(
config.Runner.Secrets,
),
secret.External(
config.Secret.Endpoint,
config.Secret.Token,
config.Secret.SkipVerify,
),
),
runner := &runtime.Runner{
Client: cli,
Machine: config.Runner.Name,
Reporter: tracer,
Lookup: resource.Lookup,
Lint: linter.New().Lint,
Match: match.Func(
config.Limit.Repos,
config.Limit.Events,
config.Limit.Trusted,
),
Compiler: &compiler.Compiler{
Clone: config.Runner.Clone,
Privileged: append(config.Runner.Privileged, compiler.Privileged...),
Networks: config.Runner.Networks,
Volumes: config.Runner.Volumes,
Resources: compiler.Resources{
Memory: config.Resources.Memory,
MemorySwap: config.Resources.MemorySwap,
CPUQuota: config.Resources.CPUQuota,
CPUPeriod: config.Resources.CPUPeriod,
CPUShares: config.Resources.CPUShares,
CPUSet: config.Resources.CPUSet,
},
Execer: runtime.NewExecer(
tracer,
remote,
engine,
config.Runner.Procs,
Environ: provider.Combine(
provider.Static(config.Runner.Environ),
provider.External(
config.Environ.Endpoint,
config.Environ.Token,
config.Environ.SkipVerify,
),
),
Registry: registry.Combine(
registry.File(
config.Docker.Config,
),
registry.External(
config.Registry.Endpoint,
config.Registry.Token,
config.Registry.SkipVerify,
),
),
Secret: secret.Combine(
secret.StaticVars(
config.Runner.Secrets,
),
secret.External(
config.Secret.Endpoint,
config.Secret.Token,
config.Secret.SkipVerify,
),
),
},
Exec: runtime.NewExecer(
tracer,
remote,
engine,
config.Runner.Procs,
).Exec,
}
poller := &poller.Poller{
Client: cli,
Dispatch: runner.Run,
Filter: &client.Filter{
Kind: resource.Kind,
Type: resource.Type,

21
command/exec.go
View File

@@ -18,7 +18,6 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/compiler"
"github.com/drone-runners/drone-runner-docker/engine/linter"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/runtime"
"github.com/drone/drone-go/drone"
"github.com/drone/envsubst"
"github.com/drone/runner-go/environ"
@@ -26,7 +25,8 @@ import (
"github.com/drone/runner-go/logger"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline"
"github.com/drone/runner-go/pipeline/console"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/pipeline/streamer/console"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
"github.com/drone/signal"
@@ -102,7 +102,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
// a configuration can contain multiple pipelines.
// get a specific pipeline resource for execution.
resource, err := resource.Lookup(c.Stage.Name, manifest)
res, err := resource.Lookup(c.Stage.Name, manifest)
if err != nil {
return err
}
@@ -110,8 +110,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
// lint the pipeline and return an error if any
// linting rules are broken
lint := linter.New()
opts := linter.Opts{Trusted: c.Repo.Trusted}
err = lint.Lint(resource, opts)
err = lint.Lint(res, c.Repo)
if err != nil {
return err
}
@@ -137,8 +136,8 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
comp.Mount, _ = os.Getwd()
}
args := compiler.Args{
Pipeline: resource,
args := runtime.CompilerArgs{
Pipeline: res,
Manifest: manifest,
Build: c.Build,
Netrc: c.Netrc,
@@ -146,7 +145,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
Stage: c.Stage,
System: c.System,
}
spec := comp.Compile(nocontext, args)
spec := comp.Compile(nocontext, args).(*engine.Spec)
// include only steps that are in the include list,
// if the list in non-empty.
@@ -161,7 +160,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
continue I
}
}
step.RunPolicy = engine.RunNever
step.RunPolicy = runtime.RunNever
}
}
@@ -175,7 +174,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
}
for _, name := range c.Exclude {
if step.Name == name {
step.RunPolicy = engine.RunNever
step.RunPolicy = runtime.RunNever
continue E
}
}
@@ -184,7 +183,7 @@ func (c *execCommand) run(*kingpin.ParseContext) error {
// create a step object for each pipeline step.
for _, step := range spec.Steps {
if step.RunPolicy == engine.RunNever {
if step.RunPolicy == runtime.RunNever {
continue
}
c.Stage.Steps = append(c.Stage.Steps, &drone.Step{

3
engine/compiler/clone.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
)
// default name of the clone step.
@@ -47,7 +48,7 @@ func createClone(src *resource.Pipeline) *engine.Step {
return &engine.Step{
Name: cloneStepName,
Image: cloneImage(src.Platform),
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
Envs: cloneParams(src.Clone),
}
}

5
engine/compiler/clone_test.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
@@ -46,7 +47,7 @@ func TestClone(t *testing.T) {
Image: "drone/git:latest",
Name: "clone",
Pull: engine.PullIfNotExists,
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
WorkingDir: "/drone/src",
Volumes: []*engine.VolumeMount{
&engine.VolumeMount{
@@ -88,7 +89,7 @@ func TestCloneCreate(t *testing.T) {
want := &engine.Step{
Name: "clone",
Image: "drone/git:latest",
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
Envs: map[string]string{"PLUGIN_DEPTH": "50"},
}
src := &resource.Pipeline{Clone: manifest.Clone{Depth: 50}}

93
engine/compiler/compiler.go
View File

@@ -12,12 +12,12 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/clone"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/labels"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/registry/auths"
"github.com/drone/runner-go/secret"
@@ -52,51 +52,9 @@ type Resources struct {
CPUSet []string
}
// Args provides compiler arguments.
type Args struct {
// Manifest provides the parsed manifest.
Manifest *manifest.Manifest
// Pipeline provides the parsed pipeline. This pipeline is
// the compiler source and is converted to the intermediate
// representation by the Compile method.
Pipeline *resource.Pipeline
// Build provides the compiler with stage information that
// is converted to environment variable format and passed to
// each pipeline step. It is also used to clone the commit.
Build *drone.Build
// Stage provides the compiler with stage information that
// is converted to environment variable format and passed to
// each pipeline step.
Stage *drone.Stage
// Repo provides the compiler with repo information. This
// repo information is converted to environment variable
// format and passed to each pipeline step. It is also used
// to clone the repository.
Repo *drone.Repo
// System provides the compiler with system information that
// is converted to environment variable format and passed to
// each pipeline step.
System *drone.System
// Netrc provides netrc parameters that can be used by the
// default clone step to authenticate to the remote
// repository.
Netrc *drone.Netrc
// Secret returns a named secret value that can be injected
// into the pipeline step.
Secret secret.Provider
}
// Compiler compiles the Yaml configuration file to an
// intermediate representation optimized for simple execution.
type Compiler struct {
// Environ provides a set of environment variables that
// should be added to each pipeline step by default.
Environ provider.Provider
@@ -139,11 +97,12 @@ type Compiler struct {
}
// Compile compiles the configuration file.
func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
os := args.Pipeline.Platform.OS
func (c *Compiler) Compile(ctx context.Context, args runtime.CompilerArgs) runtime.Spec {
pipeline := args.Pipeline.(*resource.Pipeline)
os := pipeline.Platform.OS
// create the workspace paths
base, path, full := createWorkspace(args.Pipeline)
base, path, full := createWorkspace(pipeline)
// if the source code is mounted from the host, the
// target mount path inside the container must be the
@@ -197,10 +156,10 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
Labels: labels,
},
Platform: engine.Platform{
OS: args.Pipeline.Platform.OS,
Arch: args.Pipeline.Platform.Arch,
Variant: args.Pipeline.Platform.Variant,
Version: args.Pipeline.Platform.Version,
OS: pipeline.Platform.OS,
Arch: pipeline.Platform.Arch,
Variant: pipeline.Platform.Variant,
Version: pipeline.Platform.Version,
},
Volumes: []*engine.Volume{volume},
}
@@ -215,7 +174,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
envs = environ.Combine(
envs,
args.Build.Params,
args.Pipeline.Environment,
pipeline.Environment,
environ.Proxy(),
environ.System(args.System),
environ.Repo(args.Repo),
@@ -223,8 +182,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
environ.Stage(args.Stage),
environ.Link(args.Repo, args.Build, args.System),
clone.Environ(clone.Config{
SkipVerify: args.Pipeline.Clone.SkipVerify,
Trace: args.Pipeline.Clone.Trace,
SkipVerify: pipeline.Clone.SkipVerify,
Trace: pipeline.Clone.Trace,
User: clone.User{
Name: args.Build.AuthorName,
Email: args.Build.AuthorEmail,
@@ -272,8 +231,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// create the clone step
if args.Pipeline.Clone.Disable == false {
step := createClone(args.Pipeline)
if pipeline.Clone.Disable == false {
step := createClone(pipeline)
step.ID = random()
step.Envs = environ.Combine(envs, step.Envs)
step.WorkingDir = full
@@ -291,13 +250,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the repository is mounted from a local
// volume we should disable cloning.
if c.Mount != "" {
step.RunPolicy = engine.RunNever
step.RunPolicy = runtime.RunNever
}
}
// create steps
for _, src := range args.Pipeline.Services {
dst := createStep(args.Pipeline, src)
for _, src := range pipeline.Services {
dst := createStep(pipeline, src)
dst.Detach = true
dst.Envs = environ.Combine(envs, dst.Envs)
dst.Volumes = append(dst.Volumes, mount)
@@ -309,13 +268,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the pipeline step has unmet conditions the step is
// automatically skipped.
if !src.When.Match(match) {
dst.RunPolicy = engine.RunNever
dst.RunPolicy = runtime.RunNever
}
}
// create steps
for _, src := range args.Pipeline.Steps {
dst := createStep(args.Pipeline, src)
for _, src := range pipeline.Steps {
dst := createStep(pipeline, src)
dst.Envs = environ.Combine(envs, dst.Envs)
dst.Volumes = append(dst.Volumes, mount)
dst.Labels = labels
@@ -326,7 +285,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the pipeline step has unmet conditions the step is
// automatically skipped.
if !src.When.Match(match) {
dst.RunPolicy = engine.RunNever
dst.RunPolicy = runtime.RunNever
}
// if the pipeline step has an approved image, it is
@@ -339,9 +298,9 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
if isGraph(spec) == false {
configureSerial(spec)
} else if args.Pipeline.Clone.Disable == false {
} else if pipeline.Clone.Disable == false {
configureCloneDeps(spec)
} else if args.Pipeline.Clone.Disable == true {
} else if pipeline.Clone.Disable == true {
removeCloneDeps(spec)
}
@@ -365,7 +324,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// get registry credentials from secrets
for _, name := range args.Pipeline.PullSecrets {
for _, name := range pipeline.PullSecrets {
secret, ok := c.findSecret(ctx, args, name)
if ok {
parsed, err := auths.ParseString(secret)
@@ -425,7 +384,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// append volumes
for _, v := range args.Pipeline.Volumes {
for _, v := range pipeline.Volumes {
id := random()
src := new(engine.Volume)
if v.EmptyDir != nil {
@@ -478,7 +437,7 @@ func (c *Compiler) isPrivileged(step *resource.Step) bool {
// helper function attempts to find and return the named secret.
// from the secret provider.
func (c *Compiler) findSecret(ctx context.Context, args Args, name string) (s string, ok bool) {
func (c *Compiler) findSecret(ctx context.Context, args runtime.CompilerArgs, name string) (s string, ok bool) {
if name == "" {
return
}

9
engine/compiler/compiler_test.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
@@ -65,10 +66,10 @@ func TestCompile_CloneDisabled_Graph(t *testing.T) {
// defined in the when block are not satisfied.
func TestCompile_Match(t *testing.T) {
ir := testCompile(t, "testdata/match.yml", "testdata/match.json")
if ir.Steps[0].RunPolicy != engine.RunOnSuccess {
if ir.Steps[0].RunPolicy != runtime.RunOnSuccess {
t.Errorf("Expect run on success")
}
if ir.Steps[1].RunPolicy != engine.RunNever {
if ir.Steps[1].RunPolicy != runtime.RunNever {
t.Errorf("Expect run never")
}
}
@@ -77,7 +78,7 @@ func TestCompile_Match(t *testing.T) {
// success or failure are configured to always run.
func TestCompile_RunAlways(t *testing.T) {
ir := testCompile(t, "testdata/run_always.yml", "testdata/run_always.json")
if ir.Steps[0].RunPolicy != engine.RunAlways {
if ir.Steps[0].RunPolicy != runtime.RunAlways {
t.Errorf("Expect run always")
}
}
@@ -86,7 +87,7 @@ func TestCompile_RunAlways(t *testing.T) {
// are configured to run on failure.
func TestCompile_RunFailure(t *testing.T) {
ir := testCompile(t, "testdata/run_failure.yml", "testdata/run_failure.json")
if ir.Steps[0].RunPolicy != engine.RunOnFailure {
if ir.Steps[0].RunPolicy != runtime.RunOnFailure {
t.Errorf("Expect run on failure")
}
}

6
engine/compiler/step.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/encoder"
"github.com/drone/runner-go/pipeline/runtime"
)
func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step {
@@ -94,9 +96,9 @@ func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step {
// success by default, but may be optionally configured
// to run on failure.
if isRunAlways(src) {
dst.RunPolicy = engine.RunAlways
dst.RunPolicy = runtime.RunAlways
} else if isRunOnFailure(src) {
dst.RunPolicy = engine.RunOnFailure
dst.RunPolicy = runtime.RunOnFailure
}
return dst

55
engine/const.go
View File

@@ -61,58 +61,3 @@ func (p *PullPolicy) UnmarshalJSON(b []byte) error {
*p = pullPolicyName[s]
return nil
}
// RunPolicy defines the policy for starting containers
// based on the point-in-time pass or fail state of
// the pipeline.
type RunPolicy int
// RunPolicy enumeration.
const (
RunOnSuccess RunPolicy = iota
RunOnFailure
RunAlways
RunNever
)
func (r RunPolicy) String() string {
return runPolicyID[r]
}
var runPolicyID = map[RunPolicy]string{
RunOnSuccess: "on-success",
RunOnFailure: "on-failure",
RunAlways: "always",
RunNever: "never",
}
var runPolicyName = map[string]RunPolicy{
"": RunOnSuccess,
"on-success": RunOnSuccess,
"on-failure": RunOnFailure,
"always": RunAlways,
"never": RunNever,
}
// MarshalJSON marshals the string representation of the
// run type to JSON.
func (r *RunPolicy) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString(`"`)
buffer.WriteString(runPolicyID[*r])
buffer.WriteString(`"`)
return buffer.Bytes(), nil
}
// UnmarshalJSON unmarshals the json representation of the
// run type from a string value.
func (r *RunPolicy) UnmarshalJSON(b []byte) error {
// unmarshal as string
var s string
err := json.Unmarshal(b, &s)
if err != nil {
return err
}
// lookup value
*r = runPolicyName[s]
return nil
}

115
engine/const_test.go
View File

@@ -10,121 +10,6 @@ import (
"testing"
)
//
// runtime policy unit tests.
//
func TestRunPolicy_Marshal(t *testing.T) {
tests := []struct {
policy RunPolicy
data string
}{
{
policy: RunAlways,
data: `"always"`,
},
{
policy: RunOnFailure,
data: `"on-failure"`,
},
{
policy: RunOnSuccess,
data: `"on-success"`,
},
{
policy: RunNever,
data: `"never"`,
},
}
for _, test := range tests {
data, err := json.Marshal(&test.policy)
if err != nil {
t.Error(err)
return
}
if bytes.Equal([]byte(test.data), data) == false {
t.Errorf("Failed to marshal policy %s", test.policy)
}
}
}
func TestRunPolicy_Unmarshal(t *testing.T) {
tests := []struct {
policy RunPolicy
data string
}{
{
policy: RunAlways,
data: `"always"`,
},
{
policy: RunOnFailure,
data: `"on-failure"`,
},
{
policy: RunOnSuccess,
data: `"on-success"`,
},
{
policy: RunNever,
data: `"never"`,
},
{
// no policy should default to on-success
policy: RunOnSuccess,
data: `""`,
},
}
for _, test := range tests {
var policy RunPolicy
err := json.Unmarshal([]byte(test.data), &policy)
if err != nil {
t.Error(err)
return
}
if got, want := policy, test.policy; got != want {
t.Errorf("Want policy %q, got %q", want, got)
}
}
}
func TestRunPolicy_UnmarshalTypeError(t *testing.T) {
var policy RunPolicy
err := json.Unmarshal([]byte("[]"), &policy)
if _, ok := err.(*json.UnmarshalTypeError); !ok {
t.Errorf("Expect unmarshal error return when JSON invalid")
}
}
func TestRunPolicy_String(t *testing.T) {
tests := []struct {
policy RunPolicy
value string
}{
{
policy: RunAlways,
value: "always",
},
{
policy: RunOnFailure,
value: "on-failure",
},
{
policy: RunOnSuccess,
value: "on-success",
},
}
for _, test := range tests {
if got, want := test.policy.String(), test.value; got != want {
t.Errorf("Want policy string %q, got %q", want, got)
}
}
}
//
// pull policy unit tests.
//
func TestPullPolicy_Marshal(t *testing.T) {
tests := []struct {
policy PullPolicy

293
engine/engine.go
View File

@@ -7,17 +7,288 @@ package engine
import (
"context"
"io"
"io/ioutil"
"github.com/drone-runners/drone-runner-docker/internal/docker/errors"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage"
"github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry/auths"
"docker.io/go-docker"
"docker.io/go-docker/api/types"
"docker.io/go-docker/api/types/network"
"docker.io/go-docker/api/types/volume"
)
// Engine is the interface that must be implemented by a
// pipeline execution engine.
type Engine interface {
// Setup the pipeline environment.
Setup(context.Context, *Spec) error
// Destroy the pipeline environment.
Destroy(context.Context, *Spec) error
// Run runs the pipeine step.
Run(context.Context, *Spec, *Step, io.Writer) (*State, error)
// Opts configures the Docker engine.
type Opts struct {
HidePull bool
}
// Docker implements a Docker pipeline engine.
type Docker struct {
client docker.APIClient
hidePull bool
}
// New returns a new engine.
func New(client docker.APIClient, opts Opts) *Docker {
return &Docker{
client: client,
hidePull: opts.HidePull,
}
}
// NewEnv returns a new Engine from the environment.
func NewEnv(opts Opts) (*Docker, error) {
cli, err := docker.NewEnvClient()
if err != nil {
return nil, err
}
return New(cli, opts), nil
}
// Ping pings the Docker daemon.
func (e *Docker) Ping(ctx context.Context) error {
_, err := e.client.Ping(ctx)
return err
}
// Setup the pipeline environment.
func (e *Docker) Setup(ctx context.Context, specv runtime.Spec) error {
spec := specv.(*Spec)
// creates the default temporary (local) volumes
// that are mounted into each container step.
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
_, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{
Name: vol.EmptyDir.ID,
Driver: "local",
Labels: vol.EmptyDir.Labels,
})
if err != nil {
return errors.TrimExtraInfo(err)
}
}
// creates the default pod network. All containers
// defined in the pipeline are attached to this network.
driver := "bridge"
if spec.Platform.OS == "windows" {
driver = "nat"
}
_, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{
Driver: driver,
Labels: spec.Network.Labels,
})
return errors.TrimExtraInfo(err)
}
// Destroy the pipeline environment.
func (e *Docker) Destroy(ctx context.Context, specv runtime.Spec) error {
spec := specv.(*Spec)
removeOpts := types.ContainerRemoveOptions{
Force: true,
RemoveLinks: false,
RemoveVolumes: true,
}
// stop all containers
for _, step := range spec.Steps {
e.client.ContainerKill(ctx, step.ID, "9")
}
// cleanup all containers
for _, step := range spec.Steps {
e.client.ContainerRemove(ctx, step.ID, removeOpts)
}
// cleanup all volumes
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
// tempfs volumes do not have a volume entry,
// and therefore do not require removal.
if vol.EmptyDir.Medium == "memory" {
continue
}
e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true)
}
// cleanup the network
e.client.NetworkRemove(ctx, spec.Network.ID)
// notice that we never collect or return any errors.
// this is because we silently ignore cleanup failures
// and instead ask the system admin to periodically run
// `docker prune` commands.
return nil
}
// Run runs the pipeline step.
func (e *Docker) Run(ctx context.Context, specv runtime.Spec, stepv runtime.Step, output io.Writer) (*runtime.State, error) {
spec := specv.(*Spec)
step := stepv.(*Step)
// create the container
err := e.create(ctx, spec, step, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// start the container
err = e.start(ctx, step.ID)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// tail the container
err = e.tail(ctx, step.ID, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// wait for the response
return e.wait(ctx, step.ID)
}
//
// emulate docker commands
//
func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error {
// create pull options with encoded authorization credentials.
pullopts := types.ImagePullOptions{}
if step.Auth != nil {
pullopts.RegistryAuth = auths.Header(
step.Auth.Username,
step.Auth.Password,
)
}
// automatically pull the latest version of the image if requested
// by the process configuration, or if the image is :latest
if step.Pull == PullAlways ||
(step.Pull == PullDefault && image.IsLatest(step.Image)) {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr == nil {
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
}
if pullerr != nil {
return pullerr
}
}
_, err := e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
// automatically pull and try to re-create the image if the
// failure is caused because the image does not exist.
if docker.IsErrImageNotFound(err) && step.Pull != PullNever {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr != nil {
return pullerr
}
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
// once the image is successfully pulled we attempt to
// re-create the container.
_, err = e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
}
if err != nil {
return err
}
// attach the container to user-defined networks.
// primarily used to attach global user-defined networks.
if step.Network == "" {
for _, net := range step.Networks {
err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{
Aliases: []string{net},
})
if err != nil {
return nil
}
}
}
return nil
}
// helper function emulates the `docker start` command.
func (e *Docker) start(ctx context.Context, id string) error {
return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{})
}
// helper function emulates the `docker wait` command, blocking
// until the container stops and returning the exit code.
func (e *Docker) wait(ctx context.Context, id string) (*runtime.State, error) {
wait, errc := e.client.ContainerWait(ctx, id, "")
select {
case <-wait:
case <-errc:
}
info, err := e.client.ContainerInspect(ctx, id)
if err != nil {
return nil, err
}
if info.State.Running {
// TODO(bradrydewski) if the state is still running
// we should call wait again.
}
return &runtime.State{
Exited: true,
ExitCode: info.State.ExitCode,
OOMKilled: info.State.OOMKilled,
}, nil
}
// helper function emulates the `docker logs -f` command, streaming
// all container logs until the container stops.
func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error {
opts := types.ContainerLogsOptions{
Follow: true,
ShowStdout: true,
ShowStderr: true,
Details: false,
Timestamps: false,
}
logs, err := e.client.ContainerLogs(ctx, id, opts)
if err != nil {
return err
}
go func() {
stdcopy.StdCopy(output, output, logs)
logs.Close()
}()
return nil
}

286
engine/engine_impl.go
View File

@@ -1,286 +0,0 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package engine
import (
"context"
"io"
"io/ioutil"
"github.com/drone-runners/drone-runner-docker/internal/docker/errors"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage"
"github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy"
"github.com/drone/runner-go/registry/auths"
"docker.io/go-docker"
"docker.io/go-docker/api/types"
"docker.io/go-docker/api/types/network"
"docker.io/go-docker/api/types/volume"
)
// Opts configures the Docker engine.
type Opts struct {
HidePull bool
}
// Docker implements a Docker pipeline engine.
type Docker struct {
client docker.APIClient
hidePull bool
}
// New returns a new engine.
func New(client docker.APIClient, opts Opts) *Docker {
return &Docker{
client: client,
hidePull: opts.HidePull,
}
}
// NewEnv returns a new Engine from the environment.
func NewEnv(opts Opts) (*Docker, error) {
cli, err := docker.NewEnvClient()
if err != nil {
return nil, err
}
return New(cli, opts), nil
}
// Ping pings the Docker daemon.
func (e *Docker) Ping(ctx context.Context) error {
_, err := e.client.Ping(ctx)
return err
}
// Setup the pipeline environment.
func (e *Docker) Setup(ctx context.Context, spec *Spec) error {
// creates the default temporary (local) volumes
// that are mounted into each container step.
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
_, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{
Name: vol.EmptyDir.ID,
Driver: "local",
Labels: vol.EmptyDir.Labels,
})
if err != nil {
return errors.TrimExtraInfo(err)
}
}
// creates the default pod network. All containers
// defined in the pipeline are attached to this network.
driver := "bridge"
if spec.Platform.OS == "windows" {
driver = "nat"
}
_, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{
Driver: driver,
Labels: spec.Network.Labels,
})
return errors.TrimExtraInfo(err)
}
// Destroy the pipeline environment.
func (e *Docker) Destroy(ctx context.Context, spec *Spec) error {
removeOpts := types.ContainerRemoveOptions{
Force: true,
RemoveLinks: false,
RemoveVolumes: true,
}
// stop all containers
for _, step := range spec.Steps {
e.client.ContainerKill(ctx, step.ID, "9")
}
// cleanup all containers
for _, step := range spec.Steps {
e.client.ContainerRemove(ctx, step.ID, removeOpts)
}
// cleanup all volumes
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
// tempfs volumes do not have a volume entry,
// and therefore do not require removal.
if vol.EmptyDir.Medium == "memory" {
continue
}
e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true)
}
// cleanup the network
e.client.NetworkRemove(ctx, spec.Network.ID)
// notice that we never collect or return any errors.
// this is because we silently ignore cleanup failures
// and instead ask the system admin to periodically run
// `docker prune` commands.
return nil
}
// Run runs the pipeline step.
func (e *Docker) Run(ctx context.Context, spec *Spec, step *Step, output io.Writer) (*State, error) {
// create the container
err := e.create(ctx, spec, step, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// start the container
err = e.start(ctx, step.ID)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// tail the container
err = e.tail(ctx, step.ID, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// wait for the response
return e.wait(ctx, step.ID)
}
//
// emulate docker commands
//
func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error {
// create pull options with encoded authorization credentials.
pullopts := types.ImagePullOptions{}
if step.Auth != nil {
pullopts.RegistryAuth = auths.Header(
step.Auth.Username,
step.Auth.Password,
)
}
// automatically pull the latest version of the image if requested
// by the process configuration, or if the image is :latest
if step.Pull == PullAlways ||
(step.Pull == PullDefault && image.IsLatest(step.Image)) {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr == nil {
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
}
if pullerr != nil {
return pullerr
}
}
_, err := e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
// automatically pull and try to re-create the image if the
// failure is caused because the image does not exist.
if docker.IsErrImageNotFound(err) && step.Pull != PullNever {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr != nil {
return pullerr
}
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
// once the image is successfully pulled we attempt to
// re-create the container.
_, err = e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
}
if err != nil {
return err
}
// attach the container to user-defined networks.
// primarily used to attach global user-defined networks.
if step.Network == "" {
for _, net := range step.Networks {
err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{
Aliases: []string{net},
})
if err != nil {
return nil
}
}
}
return nil
}
// helper function emulates the `docker start` command.
func (e *Docker) start(ctx context.Context, id string) error {
return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{})
}
// helper function emulates the `docker wait` command, blocking
// until the container stops and returning the exit code.
func (e *Docker) wait(ctx context.Context, id string) (*State, error) {
wait, errc := e.client.ContainerWait(ctx, id, "")
select {
case <-wait:
case <-errc:
}
info, err := e.client.ContainerInspect(ctx, id)
if err != nil {
return nil, err
}
if info.State.Running {
// TODO(bradrydewski) if the state is still running
// we should call wait again.
}
return &State{
Exited: true,
ExitCode: info.State.ExitCode,
OOMKilled: info.State.OOMKilled,
}, nil
}
// helper function emulates the `docker logs -f` command, streaming
// all container logs until the container stops.
func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error {
opts := types.ContainerLogsOptions{
Follow: true,
ShowStdout: true,
ShowStderr: true,
Details: false,
Timestamps: false,
}
logs, err := e.client.ContainerLogs(ctx, id, opts)
if err != nil {
return err
}
go func() {
stdcopy.StdCopy(output, output, logs)
logs.Close()
}()
return nil
}

6
engine/linter/linter.go
View File

@@ -11,6 +11,8 @@ import (
"strings"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/manifest"
)
// ErrDuplicateStepName is returned when two Pipeline steps
@@ -43,8 +45,8 @@ func New() *Linter {
// Lint executes the linting rules for the pipeline
// configuration.
func (l *Linter) Lint(pipeline *resource.Pipeline, opts Opts) error {
return checkPipeline(pipeline, opts.Trusted)
func (l *Linter) Lint(pipeline manifest.Resource, repo *drone.Repo) error {
return checkPipeline(pipeline.(*resource.Pipeline), repo.Trusted)
}
func checkPipeline(pipeline *resource.Pipeline, trusted bool) error {

3
engine/linter/linter_test.go
View File

@@ -9,6 +9,7 @@ import (
"testing"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/manifest"
)
@@ -212,7 +213,7 @@ func TestLint(t *testing.T) {
}
lint := New()
opts := Opts{Trusted: test.trusted}
opts := &drone.Repo{Trusted: test.trusted}
err = lint.Lint(resources.Resources[0].(*resource.Pipeline), opts)
if err == nil && test.invalid == true {
t.Logf("yaml: %s", test.path)

56
engine/replacer/replacer.go
View File

@@ -1,56 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package replacer
import (
"fmt"
"io"
"strings"
"github.com/drone-runners/drone-runner-docker/engine"
)
const maskedf = "[secret:%s]"
// Replacer is an io.Writer that finds and masks sensitive data.
type Replacer struct {
w io.WriteCloser
r *strings.Replacer
}
// New returns a replacer that wraps writer w.
func New(w io.WriteCloser, secrets []*engine.Secret) io.WriteCloser {
var oldnew []string
for _, secret := range secrets {
if len(secret.Data) == 0 || secret.Mask == false {
continue
}
name := strings.ToLower(secret.Name)
masked := fmt.Sprintf(maskedf, name)
oldnew = append(oldnew, string(secret.Data))
oldnew = append(oldnew, masked)
}
if len(oldnew) == 0 {
return w
}
return &Replacer{
w: w,
r: strings.NewReplacer(oldnew...),
}
}
// Write writes p to the base writer. The method scans for any
// sensitive data in p and masks before writing.
func (r *Replacer) Write(p []byte) (n int, err error) {
_, err = r.w.Write([]byte(r.r.Replace(string(p))))
return len(p), err
}
// Close closes the base writer.
func (r *Replacer) Close() error {
return r.w.Close()
}

56
engine/replacer/replacer_test.go
View File

@@ -1,56 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package replacer
import (
"bytes"
"io"
"testing"
"github.com/drone-runners/drone-runner-docker/engine"
)
func TestReplace(t *testing.T) {
secrets := []*engine.Secret{
{Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false},
{Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: true},
{Name: "DOCKER_EMAIL", Data: []byte(""), Mask: true},
}
buf := new(bytes.Buffer)
w := New(&nopCloser{buf}, secrets)
w.Write([]byte("username octocat password correct-horse-batter-staple"))
w.Close()
if got, want := buf.String(), "username octocat password [secret:docker_password]"; got != want {
t.Errorf("Want masked string %s, got %s", want, got)
}
}
// this test verifies that if there are no secrets to scan and
// mask, the io.WriteCloser is returned as-is.
func TestReplaceNone(t *testing.T) {
secrets := []*engine.Secret{
{Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false},
{Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: false},
}
buf := new(bytes.Buffer)
w := &nopCloser{buf}
r := New(w, secrets)
if w != r {
t.Errorf("Expect buffer returned with no replacer")
}
}
type nopCloser struct {
io.Writer
}
func (*nopCloser) Close() error {
return nil
}

2
engine/resource/lookup.go
View File

@@ -11,7 +11,7 @@ import (
)
// Lookup returns the named pipeline from the Manifest.
func Lookup(name string, manifest *manifest.Manifest) (*Pipeline, error) {
func Lookup(name string, manifest *manifest.Manifest) (manifest.Resource, error) {
for _, resource := range manifest.Resources {
if !isNameMatch(resource.GetName(), name) {
continue

62
engine/spec.go
View File

@@ -4,7 +4,13 @@
package engine
import (
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/pipeline/runtime"
)
type (
// Spec provides the pipeline spec. This provides the
// required instructions for reproducible pipeline
// execution.
@@ -44,7 +50,7 @@ type (
Networks []string `json:"networks,omitempty"`
Privileged bool `json:"privileged,omitempty"`
Pull PullPolicy `json:"pull,omitempty"`
RunPolicy RunPolicy `json:"run_policy,omitempty"`
RunPolicy runtime.RunPolicy `json:"run_policy,omitempty"`
Secrets []*Secret `json:"secrets,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
User string `json:"user,omitempty"`
@@ -52,14 +58,6 @@ type (
WorkingDir string `json:"working_dir,omitempty"`
}
// Platform defines the target platform.
Platform struct {
OS string `json:"os,omitempty"`
Arch string `json:"arch,omitempty"`
Variant string `json:"variant,omitempty"`
Version string `json:"version,omitempty"`
}
// Secret represents a secret variable.
Secret struct {
Name string `json:"name,omitempty"`
@@ -68,11 +66,12 @@ type (
Mask bool `json:"mask,omitempty"`
}
// State represents the process state.
State struct {
ExitCode int // Container exit code
Exited bool // Container exited
OOMKilled bool // Container is oom killed
// Platform defines the target platform.
Platform struct {
OS string `json:"os,omitempty"`
Arch string `json:"arch,omitempty"`
Variant string `json:"variant,omitempty"`
Version string `json:"version,omitempty"`
}
// Volume that can be mounted by containers.
@@ -128,3 +127,38 @@ type (
Password string `json:"password,omitempty"`
}
)
//
// implements the Spec interface
//
func (s *Spec) StepLen() int { return len(s.Steps) }
func (s *Spec) StepAt(i int) runtime.Step { return s.Steps[i] }
//
// implements the Secret interface
//
func (s *Secret) GetName() string { return s.Name }
func (s *Secret) GetValue() string { return string(s.Data) }
func (s *Secret) IsMasked() bool { return s.Mask }
//
// implements the Step interface
//
func (s *Step) GetName() string { return s.Name }
func (s *Step) GetDependencies() []string { return s.DependsOn }
func (s *Step) GetEnviron() map[string]string { return s.Envs }
func (s *Step) SetEnviron(env map[string]string) { s.Envs = env }
func (s *Step) GetErrPolicy() runtime.ErrPolicy { return runtime.ErrFail }
func (s *Step) GetRunPolicy() runtime.RunPolicy { return s.RunPolicy }
func (s *Step) GetSecretAt(i int) runtime.Secret { return s.Secrets[i] }
func (s *Step) GetSecretLen() int { return len(s.Secrets) }
func (s *Step) IsDetached() bool { return s.Detach }
func (s *Step) Clone() runtime.Step {
dst := new(Step)
*dst = *s
dst.Envs = environ.Combine(s.Envs)
return dst
}

8
go.mod
View File

@@ -4,19 +4,17 @@ go 1.12
require (
docker.io/go-docker v1.0.0
github.com/Microsoft/go-winio v0.4.11 // indirect
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
github.com/buildkite/yaml v2.1.0+incompatible
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/go-connections v0.3.0 // indirect
github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79
github.com/drone/drone-runtime v1.1.0
github.com/drone/envsubst v1.0.2
github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7
github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c
github.com/drone/signal v1.0.0
github.com/ghodss/yaml v1.0.0
github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 // indirect
github.com/golang/mock v1.3.1
github.com/google/go-cmp v0.3.0
github.com/hashicorp/go-multierror v1.0.0
@@ -25,8 +23,6 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/mattn/go-isatty v0.0.8
github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/pkg/errors v0.8.1 // indirect
github.com/sirupsen/logrus v1.4.2
github.com/stretchr/testify v1.3.0 // indirect

42
go.sum
View File

@@ -21,10 +21,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU=
github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4=
github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba h1:GKiT4UPBligLXJAP1zRllHvTUygAAlgS3t9LM9aasp0=
@@ -32,6 +34,8 @@ github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba/go.mod h1:GxyeGCl
github.com/drone/drone-go v1.1.0 h1:2mritc5b7PhQWvILNyzaImZMRWVbMmmZ5Q0UDwwO7SI=
github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79 h1:jW+dJ8HrZ1CbazlsYoriOOCQnVJ2NkfNczLHs6UMU6I=
github.com/drone/drone-go v1.1.1-0.20191119212130-1d2e07e87e79/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4=
github.com/drone/drone-runtime v1.1.0 h1:IsKbwiLY6+ViNBzX0F8PERJVZZcEJm9rgxEh3uZP5IE=
github.com/drone/drone-runtime v1.1.0/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs=
github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo=
github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
github.com/drone/runner-go v1.2.2 h1:fwYgjyJl6KdjQGEUFof9+HLtNpK3iHq7UuR+/aYNyDk=
@@ -46,23 +50,39 @@ github.com/drone/runner-go v1.3.0/go.mod h1:61VgQWhZbNPXp01lBuR7PAztTMySGLnMzK/4
github.com/drone/runner-go v1.4.0 h1:zAeYtlKQGvJr2ehfLzQWblzWzvfdkNaapbr6x536fLA=
github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7 h1:iNLp8xT0rMcV/tT2J3fCuEbWLqoP7CiL3WR8W3i3HpQ=
github.com/drone/runner-go v1.4.1-0.20191119212738-c0d9268011a7/go.mod h1:IqwuMbIoeH45k4NemcNPwymm+l386EeyBC166UElURw=
github.com/drone/runner-go v1.4.1-0.20191206074106-b546cbadce98 h1:281h7RjcgHLOsqP0abNrS8ESVhtNks2i/WWW0+S0HWw=
github.com/drone/runner-go v1.4.1-0.20191206074106-b546cbadce98/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o=
github.com/drone/runner-go v1.4.1-0.20191206204642-f16a481b8b8f h1:TC1w5LWTOEbbatXnx92vRqsEsiIenzwWMmIqEvYqo+U=
github.com/drone/runner-go v1.4.1-0.20191206204642-f16a481b8b8f/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o=
github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c h1:T3CIdRKiTiaNbInLmtPOoY/9BNUPaL2ipBr2jHzaJsE=
github.com/drone/runner-go v1.4.1-0.20191206210533-80bb9688cb0c/go.mod h1:ZIhsNU4EHG7R7J+OXeXuwwAxlxOBYLCc0gCkbbhWb/o=
github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI=
github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 h1:zDlw+wgyXdfkRuvFCdEDUiPLmZp2cvf/dWHazY0a5VM=
github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
@@ -72,33 +92,47 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 h1:dnMxwus89s86tI8rcGVp2HwZzlz7c5o92VOy7dSckBQ=
github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4/go.mod h1:cojhOHk1gbMeklOyDP2oKKLftefXoJreOQGOrXk+Z38=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -106,11 +140,19 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

242
runtime/execer.go
View File

@@ -1,242 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime
import (
"context"
"sync"
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/replacer"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/logger"
"github.com/drone/runner-go/pipeline"
"github.com/hashicorp/go-multierror"
"github.com/natessilva/dag"
"golang.org/x/sync/semaphore"
)
// Execer is the execution context for executing the intermediate
// representation of a pipeline.
type Execer interface {
Exec(context.Context, *engine.Spec, *pipeline.State) error
}
type execer struct {
mu sync.Mutex
engine engine.Engine
reporter pipeline.Reporter
streamer pipeline.Streamer
sem *semaphore.Weighted
}
// NewExecer returns a new execer used
func NewExecer(
reporter pipeline.Reporter,
streamer pipeline.Streamer,
engine engine.Engine,
procs int64,
) Execer {
exec := &execer{
reporter: reporter,
streamer: streamer,
engine: engine,
}
if procs > 0 {
// optional semaphor that limits the number of steps
// that can execute concurrently.
exec.sem = semaphore.NewWeighted(procs)
}
return exec
}
// Exec executes the intermediate representation of the pipeline
// and returns an error if execution fails.
func (e *execer) Exec(ctx context.Context, spec *engine.Spec, state *pipeline.State) error {
defer e.engine.Destroy(noContext, spec)
if err := e.engine.Setup(noContext, spec); err != nil {
state.FailAll(err)
return e.reporter.ReportStage(noContext, state)
}
// create a directed graph, where each vertex in the graph
// is a pipeline step.
var d dag.Runner
for _, s := range spec.Steps {
step := s
d.AddVertex(step.Name, func() error {
return e.exec(ctx, state, spec, step)
})
}
// create the vertex edges from the values configured in the
// depends_on attribute.
for _, s := range spec.Steps {
for _, dep := range s.DependsOn {
d.AddEdge(dep, s.Name)
}
}
var result error
if err := d.Run(); err != nil {
multierror.Append(result, err)
}
// once pipeline execution completes, notify the state
// manageer that all steps are finished.
state.FinishAll()
if err := e.reporter.ReportStage(noContext, state); err != nil {
multierror.Append(result, err)
}
return result
}
func (e *execer) exec(ctx context.Context, state *pipeline.State, spec *engine.Spec, step *engine.Step) error {
var result error
select {
case <-ctx.Done():
state.Cancel()
return nil
default:
}
log := logger.FromContext(ctx)
log = log.WithField("step.name", step.Name)
ctx = logger.WithContext(ctx, log)
if e.sem != nil {
// the semaphore limits the number of steps that can run
// concurrently. acquire the semaphore and release when
// the pipeline completes.
if err := e.sem.Acquire(ctx, 1); err != nil {
return nil
}
defer func() {
// recover from a panic to ensure the semaphore is
// released to prevent deadlock. we do not expect a
// panic, however, we are being overly cautious.
if r := recover(); r != nil {
// TODO(bradrydzewsi) log the panic.
}
// release the semaphore
e.sem.Release(1)
}()
}
switch {
case state.Skipped():
return nil
case state.Cancelled():
return nil
case step.RunPolicy == engine.RunNever:
return nil
case step.RunPolicy == engine.RunAlways:
break
case step.RunPolicy == engine.RunOnFailure && state.Failed() == false:
state.Skip(step.Name)
return e.reporter.ReportStep(noContext, state, step.Name)
case step.RunPolicy == engine.RunOnSuccess && state.Failed():
state.Skip(step.Name)
return e.reporter.ReportStep(noContext, state, step.Name)
}
state.Start(step.Name)
err := e.reporter.ReportStep(noContext, state, step.Name)
if err != nil {
return err
}
copy := cloneStep(step)
// the pipeline environment variables need to be updated to
// reflect the current state of the build and stage.
state.Lock()
copy.Envs = environ.Combine(
copy.Envs,
environ.Build(state.Build),
environ.Stage(state.Stage),
environ.Step(findStep(state, step.Name)),
)
state.Unlock()
// writer used to stream build logs.
wc := e.streamer.Stream(noContext, state, step.Name)
wc = replacer.New(wc, step.Secrets)
// if the step is configured as a daemon, it is detached
// from the main process and executed separately.
// todo(bradrydzewski) this code is still experimental.
if step.Detach {
go func() {
e.engine.Run(ctx, spec, copy, wc)
wc.Close()
}()
return nil
}
exited, err := e.engine.Run(ctx, spec, copy, wc)
// close the stream. If the session is a remote session, the
// full log buffer is uploaded to the remote server.
if err := wc.Close(); err != nil {
multierror.Append(result, err)
}
if exited != nil {
state.Finish(step.Name, exited.ExitCode)
err := e.reporter.ReportStep(noContext, state, step.Name)
if err != nil {
multierror.Append(result, err)
}
// if the exit code is 78 the system will skip all
// subsequent pending steps in the pipeline.
if exited.ExitCode == 78 {
state.SkipAll()
}
return result
}
switch err {
case context.Canceled, context.DeadlineExceeded:
state.Cancel()
return nil
}
// if the step failed with an internal error (as oppsed to a
// runtime error) the step is failed.
state.Fail(step.Name, err)
err = e.reporter.ReportStep(noContext, state, step.Name)
if err != nil {
multierror.Append(result, err)
}
return result
}
// helper function to clone a step. The runner mutates a step to
// update the environment variables to reflect the current
// pipeline state.
func cloneStep(src *engine.Step) *engine.Step {
dst := new(engine.Step)
*dst = *src
dst.Envs = environ.Combine(src.Envs)
return dst
}
// helper function returns the named step from the state.
func findStep(state *pipeline.State, name string) *drone.Step {
for _, step := range state.Stage.Steps {
if step.Name == name {
return step
}
}
panic("step not found: " + name)
}

39
runtime/execer_test.go
View File

@@ -1,39 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime
import (
"testing"
)
func TestExec(t *testing.T) {
t.Skip()
}
func TestExec_NonZeroExit(t *testing.T) {
t.Skip()
}
func TestExec_Exit78(t *testing.T) {
t.Skip()
}
func TestExec_Error(t *testing.T) {
t.Skip()
}
func TestExec_CtxError(t *testing.T) {
t.Skip()
}
func TestExec_ReportError(t *testing.T) {
t.Skip()
}
func TestExec_SkipCtxDone(t *testing.T) {
t.Skip()
}

76
runtime/poller.go
View File

@@ -1,76 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime
import (
"context"
"sync"
"github.com/drone/runner-go/client"
"github.com/drone/runner-go/logger"
)
var noContext = context.Background()
// Poller polls the server for pending stages and dispatches
// for execution by the Runner.
type Poller struct {
Client client.Client
Filter *client.Filter
Runner *Runner
}
// Poll opens N connections to the server to poll for pending
// stages for execution. Pending stages are dispatched to a
// Runner for execution.
func (p *Poller) Poll(ctx context.Context, n int) {
var wg sync.WaitGroup
for i := 0; i < n; i++ {
wg.Add(1)
go func(i int) {
for {
select {
case <-ctx.Done():
wg.Done()
return
default:
p.poll(ctx, i+1)
}
}
}(i)
}
wg.Wait()
}
// poll requests a stage for execution from the server, and then
// dispatches for execution.
func (p *Poller) poll(ctx context.Context, thread int) error {
log := logger.FromContext(ctx).WithField("thread", thread)
log.WithField("thread", thread).Debug("request stage from remote server")
// request a new build stage for execution from the central
// build server.
stage, err := p.Client.Request(ctx, p.Filter)
if err == context.Canceled || err == context.DeadlineExceeded {
log.WithError(err).Trace("no stage returned")
return nil
}
if err != nil {
log.WithError(err).Error("cannot request stage")
return err
}
// exit if a nil or empty stage is returned from the system
// and allow the runner to retry.
if stage == nil || stage.ID == 0 {
return nil
}
return p.Runner.Run(
logger.WithContext(noContext, log), stage)
}

27
runtime/poller_test.go
View File

@@ -1,27 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime
import (
"testing"
)
func TestPoll(t *testing.T) {
t.Skip()
}
func TestPoll_NilStage(t *testing.T) {
t.Skip()
}
func TestPoll_EmptyStage(t *testing.T) {
t.Skip()
}
func TestPoll_RequestError(t *testing.T) {
t.Skip()
}

249
runtime/runner.go
View File

@@ -1,249 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/compiler"
"github.com/drone-runners/drone-runner-docker/engine/linter"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/drone-go/drone"
"github.com/drone/envsubst"
"github.com/drone/runner-go/client"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/logger"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline"
"github.com/drone/runner-go/secret"
)
// Runner runs the pipeline.
type Runner struct {
// Client is the remote client responsible for interacting
// with the central server.
Client client.Client
// Compiler is responsible for compiling the pipeline
// configuration to the intermediate representation.
Compiler *compiler.Compiler
// Execer is responsible for executing intermediate
// representation of the pipeline and returns its results.
Execer Execer
// Linter is responsible for linting the pipeline
// and failing if any rules are broken.
Linter *linter.Linter
// Machine provides the runner with the name of the host
// machine executing the pipeline.
Machine string
// Match is an optional function that returns true if the
// repository or build match user-defined criteria. This is
// intended as a security measure to prevent a runner from
// processing an unwanted pipeline.
Match func(*drone.Repo, *drone.Build) bool
// Reporter reports pipeline status and logs back to the
// remote server.
Reporter pipeline.Reporter
}
// Run runs the pipeline stage.
func (s *Runner) Run(ctx context.Context, stage *drone.Stage) error {
log := logger.FromContext(ctx).
WithField("stage.id", stage.ID).
WithField("stage.name", stage.Name).
WithField("stage.number", stage.Number)
log.Debug("stage received")
// delivery to a single agent is not guaranteed, which means
// we need confirm receipt. The first agent that confirms
// receipt of the stage can assume ownership.
stage.Machine = s.Machine
err := s.Client.Accept(ctx, stage)
if err != nil && err == client.ErrOptimisticLock {
log.Debug("stage accepted by another runner")
return nil
}
if err != nil {
log.WithError(err).Error("cannot accept stage")
return err
}
log.Debug("stage accepted")
data, err := s.Client.Detail(ctx, stage)
if err != nil {
log.WithError(err).Error("cannot get stage details")
return err
}
log = log.WithField("repo.id", data.Repo.ID).
WithField("repo.namespace", data.Repo.Namespace).
WithField("repo.name", data.Repo.Name).
WithField("build.id", data.Build.ID).
WithField("build.number", data.Build.Number)
log.Debug("stage details fetched")
ctxdone, cancel := context.WithCancel(ctx)
defer cancel()
timeout := time.Duration(data.Repo.Timeout) * time.Minute
ctxtimeout, cancel := context.WithTimeout(ctxdone, timeout)
defer cancel()
ctxcancel, cancel := context.WithCancel(ctxtimeout)
defer cancel()
// next we opens a connection to the server to watch for
// cancellation requests. If a build is cancelled the running
// stage should also be cancelled.
go func() {
done, _ := s.Client.Watch(ctxdone, data.Build.ID)
if done {
cancel()
log.Debugln("received cancellation")
} else {
log.Debugln("done listening for cancellations")
}
}()
envs := environ.Combine(
environ.System(data.System),
environ.Repo(data.Repo),
environ.Build(data.Build),
environ.Stage(stage),
environ.Link(data.Repo, data.Build, data.System),
data.Build.Params,
)
// string substitution function ensures that string
// replacement variables are escaped and quoted if they
// contain a newline character.
subf := func(k string) string {
v := envs[k]
if strings.Contains(v, "\n") {
v = fmt.Sprintf("%q", v)
}
return v
}
state := &pipeline.State{
Build: data.Build,
Stage: stage,
Repo: data.Repo,
System: data.System,
}
// evaluates whether or not the agent can process the
// pipeline. An agent may choose to reject a repository
// or build for security reasons.
if s.Match != nil && s.Match(data.Repo, data.Build) == false {
log.Error("cannot process stage, access denied")
state.FailAll(errors.New("insufficient permission to run the pipeline"))
return s.Reporter.ReportStage(noContext, state)
}
// evaluates string replacement expressions and returns an
// update configuration file string.
config, err := envsubst.Eval(string(data.Config.Data), subf)
if err != nil {
log.WithError(err).Error("cannot emulate bash substitution")
state.FailAll(err)
return s.Reporter.ReportStage(noContext, state)
}
// parse the yaml configuration file.
manifest, err := manifest.ParseString(config)
if err != nil {
log.WithError(err).Error("cannot parse configuration file")
state.FailAll(err)
return s.Reporter.ReportStage(noContext, state)
}
// find the named stage in the yaml configuration file.
resource, err := resource.Lookup(stage.Name, manifest)
if err != nil {
log.WithError(err).Error("cannot find pipeline resource")
state.FailAll(err)
return s.Reporter.ReportStage(noContext, state)
}
// lint the pipeline configuration and fail the build
// if any linting rules are broken.
err = s.Linter.Lint(resource, linter.Opts{Trusted: data.Repo.Trusted})
if err != nil {
log.WithError(err).Error("cannot accept configuration")
state.FailAll(err)
return s.Reporter.ReportStage(noContext, state)
}
secrets := secret.Combine(
secret.Static(data.Secrets),
secret.Encrypted(),
// s.Secret,
)
// compile the yaml configuration file to an intermediate
// representation, and then
args := compiler.Args{
Pipeline: resource,
Manifest: manifest,
Build: data.Build,
Stage: stage,
Repo: data.Repo,
System: data.System,
Netrc: data.Netrc,
Secret: secrets,
}
spec := s.Compiler.Compile(ctx, args)
for _, src := range spec.Steps {
// steps that are skipped are ignored and are not stored
// in the drone database, nor displayed in the UI.
if src.RunPolicy == engine.RunNever {
continue
}
stage.Steps = append(stage.Steps, &drone.Step{
Name: src.Name,
Number: len(stage.Steps) + 1,
StageID: stage.ID,
Status: drone.StatusPending,
ErrIgnore: src.IgnoreErr,
})
}
stage.Started = time.Now().Unix()
stage.Status = drone.StatusRunning
if err := s.Client.Update(ctx, stage); err != nil {
log.WithError(err).Error("cannot update stage")
return err
}
log.Debug("updated stage to running")
ctxcancel = logger.WithContext(ctxcancel, log)
err = s.Execer.Exec(ctxcancel, spec, state)
if err != nil {
log.WithError(err).Debug("stage failed")
return err
}
log.Debug("updated stage to complete")
return nil
}

7
runtime/runner_test.go
View File

@@ -1,7 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package runtime