chaos/drone-runner-podman
Archived
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

abstract polling and execution to runner-go library

Browse Source
This commit is contained in:
Brad Rydzewski
2019-12-06 16:10:27 -08:00
parent 99e80a0352
commit 2e48cd1b3b
27 changed files with 500 additions and 1394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
engine/compiler/clone.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
)
// default name of the clone step.
@@ -47,7 +48,7 @@ func createClone(src *resource.Pipeline) *engine.Step {
return &engine.Step{
Name: cloneStepName,
Image: cloneImage(src.Platform),
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
Envs: cloneParams(src.Clone),
}
}

5
engine/compiler/clone_test.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
@@ -46,7 +47,7 @@ func TestClone(t *testing.T) {
Image: "drone/git:latest",
Name: "clone",
Pull: engine.PullIfNotExists,
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
WorkingDir: "/drone/src",
Volumes: []*engine.VolumeMount{
&engine.VolumeMount{
@@ -88,7 +89,7 @@ func TestCloneCreate(t *testing.T) {
want := &engine.Step{
Name: "clone",
Image: "drone/git:latest",
RunPolicy: engine.RunAlways,
RunPolicy: runtime.RunAlways,
Envs: map[string]string{"PLUGIN_DEPTH": "50"},
}
src := &resource.Pipeline{Clone: manifest.Clone{Depth: 50}}

93
engine/compiler/compiler.go
View File

@@ -12,12 +12,12 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/clone"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/labels"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/registry/auths"
"github.com/drone/runner-go/secret"
@@ -52,51 +52,9 @@ type Resources struct {
CPUSet []string
}
// Args provides compiler arguments.
type Args struct {
// Manifest provides the parsed manifest.
Manifest *manifest.Manifest
// Pipeline provides the parsed pipeline. This pipeline is
// the compiler source and is converted to the intermediate
// representation by the Compile method.
Pipeline *resource.Pipeline
// Build provides the compiler with stage information that
// is converted to environment variable format and passed to
// each pipeline step. It is also used to clone the commit.
Build *drone.Build
// Stage provides the compiler with stage information that
// is converted to environment variable format and passed to
// each pipeline step.
Stage *drone.Stage
// Repo provides the compiler with repo information. This
// repo information is converted to environment variable
// format and passed to each pipeline step. It is also used
// to clone the repository.
Repo *drone.Repo
// System provides the compiler with system information that
// is converted to environment variable format and passed to
// each pipeline step.
System *drone.System
// Netrc provides netrc parameters that can be used by the
// default clone step to authenticate to the remote
// repository.
Netrc *drone.Netrc
// Secret returns a named secret value that can be injected
// into the pipeline step.
Secret secret.Provider
}
// Compiler compiles the Yaml configuration file to an
// intermediate representation optimized for simple execution.
type Compiler struct {
// Environ provides a set of environment variables that
// should be added to each pipeline step by default.
Environ provider.Provider
@@ -139,11 +97,12 @@ type Compiler struct {
}
// Compile compiles the configuration file.
func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
os := args.Pipeline.Platform.OS
func (c *Compiler) Compile(ctx context.Context, args runtime.CompilerArgs) runtime.Spec {
pipeline := args.Pipeline.(*resource.Pipeline)
os := pipeline.Platform.OS
// create the workspace paths
base, path, full := createWorkspace(args.Pipeline)
base, path, full := createWorkspace(pipeline)
// if the source code is mounted from the host, the
// target mount path inside the container must be the
@@ -197,10 +156,10 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
Labels: labels,
},
Platform: engine.Platform{
OS: args.Pipeline.Platform.OS,
Arch: args.Pipeline.Platform.Arch,
Variant: args.Pipeline.Platform.Variant,
Version: args.Pipeline.Platform.Version,
OS: pipeline.Platform.OS,
Arch: pipeline.Platform.Arch,
Variant: pipeline.Platform.Variant,
Version: pipeline.Platform.Version,
},
Volumes: []*engine.Volume{volume},
}
@@ -215,7 +174,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
envs = environ.Combine(
envs,
args.Build.Params,
args.Pipeline.Environment,
pipeline.Environment,
environ.Proxy(),
environ.System(args.System),
environ.Repo(args.Repo),
@@ -223,8 +182,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
environ.Stage(args.Stage),
environ.Link(args.Repo, args.Build, args.System),
clone.Environ(clone.Config{
SkipVerify: args.Pipeline.Clone.SkipVerify,
Trace: args.Pipeline.Clone.Trace,
SkipVerify: pipeline.Clone.SkipVerify,
Trace: pipeline.Clone.Trace,
User: clone.User{
Name: args.Build.AuthorName,
Email: args.Build.AuthorEmail,
@@ -272,8 +231,8 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// create the clone step
if args.Pipeline.Clone.Disable == false {
step := createClone(args.Pipeline)
if pipeline.Clone.Disable == false {
step := createClone(pipeline)
step.ID = random()
step.Envs = environ.Combine(envs, step.Envs)
step.WorkingDir = full
@@ -291,13 +250,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the repository is mounted from a local
// volume we should disable cloning.
if c.Mount != "" {
step.RunPolicy = engine.RunNever
step.RunPolicy = runtime.RunNever
}
}
// create steps
for _, src := range args.Pipeline.Services {
dst := createStep(args.Pipeline, src)
for _, src := range pipeline.Services {
dst := createStep(pipeline, src)
dst.Detach = true
dst.Envs = environ.Combine(envs, dst.Envs)
dst.Volumes = append(dst.Volumes, mount)
@@ -309,13 +268,13 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the pipeline step has unmet conditions the step is
// automatically skipped.
if !src.When.Match(match) {
dst.RunPolicy = engine.RunNever
dst.RunPolicy = runtime.RunNever
}
}
// create steps
for _, src := range args.Pipeline.Steps {
dst := createStep(args.Pipeline, src)
for _, src := range pipeline.Steps {
dst := createStep(pipeline, src)
dst.Envs = environ.Combine(envs, dst.Envs)
dst.Volumes = append(dst.Volumes, mount)
dst.Labels = labels
@@ -326,7 +285,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
// if the pipeline step has unmet conditions the step is
// automatically skipped.
if !src.When.Match(match) {
dst.RunPolicy = engine.RunNever
dst.RunPolicy = runtime.RunNever
}
// if the pipeline step has an approved image, it is
@@ -339,9 +298,9 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
if isGraph(spec) == false {
configureSerial(spec)
} else if args.Pipeline.Clone.Disable == false {
} else if pipeline.Clone.Disable == false {
configureCloneDeps(spec)
} else if args.Pipeline.Clone.Disable == true {
} else if pipeline.Clone.Disable == true {
removeCloneDeps(spec)
}
@@ -365,7 +324,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// get registry credentials from secrets
for _, name := range args.Pipeline.PullSecrets {
for _, name := range pipeline.PullSecrets {
secret, ok := c.findSecret(ctx, args, name)
if ok {
parsed, err := auths.ParseString(secret)
@@ -425,7 +384,7 @@ func (c *Compiler) Compile(ctx context.Context, args Args) *engine.Spec {
}
// append volumes
for _, v := range args.Pipeline.Volumes {
for _, v := range pipeline.Volumes {
id := random()
src := new(engine.Volume)
if v.EmptyDir != nil {
@@ -478,7 +437,7 @@ func (c *Compiler) isPrivileged(step *resource.Step) bool {
// helper function attempts to find and return the named secret.
// from the secret provider.
func (c *Compiler) findSecret(ctx context.Context, args Args, name string) (s string, ok bool) {
func (c *Compiler) findSecret(ctx context.Context, args runtime.CompilerArgs, name string) (s string, ok bool) {
if name == "" {
return
}

9
engine/compiler/compiler_test.go
View File

@@ -19,6 +19,7 @@ import (
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/environ/provider"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry"
"github.com/drone/runner-go/secret"
@@ -65,10 +66,10 @@ func TestCompile_CloneDisabled_Graph(t *testing.T) {
// defined in the when block are not satisfied.
func TestCompile_Match(t *testing.T) {
ir := testCompile(t, "testdata/match.yml", "testdata/match.json")
if ir.Steps[0].RunPolicy != engine.RunOnSuccess {
if ir.Steps[0].RunPolicy != runtime.RunOnSuccess {
t.Errorf("Expect run on success")
}
if ir.Steps[1].RunPolicy != engine.RunNever {
if ir.Steps[1].RunPolicy != runtime.RunNever {
t.Errorf("Expect run never")
}
}
@@ -77,7 +78,7 @@ func TestCompile_Match(t *testing.T) {
// success or failure are configured to always run.
func TestCompile_RunAlways(t *testing.T) {
ir := testCompile(t, "testdata/run_always.yml", "testdata/run_always.json")
if ir.Steps[0].RunPolicy != engine.RunAlways {
if ir.Steps[0].RunPolicy != runtime.RunAlways {
t.Errorf("Expect run always")
}
}
@@ -86,7 +87,7 @@ func TestCompile_RunAlways(t *testing.T) {
// are configured to run on failure.
func TestCompile_RunFailure(t *testing.T) {
ir := testCompile(t, "testdata/run_failure.yml", "testdata/run_failure.json")
if ir.Steps[0].RunPolicy != engine.RunOnFailure {
if ir.Steps[0].RunPolicy != runtime.RunOnFailure {
t.Errorf("Expect run on failure")
}
}

6
engine/compiler/step.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/encoder"
"github.com/drone/runner-go/pipeline/runtime"
)
func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step {
@@ -94,9 +96,9 @@ func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step {
// success by default, but may be optionally configured
// to run on failure.
if isRunAlways(src) {
dst.RunPolicy = engine.RunAlways
dst.RunPolicy = runtime.RunAlways
} else if isRunOnFailure(src) {
dst.RunPolicy = engine.RunOnFailure
dst.RunPolicy = runtime.RunOnFailure
}
return dst

55
engine/const.go
View File

@@ -61,58 +61,3 @@ func (p *PullPolicy) UnmarshalJSON(b []byte) error {
*p = pullPolicyName[s]
return nil
}
// RunPolicy defines the policy for starting containers
// based on the point-in-time pass or fail state of
// the pipeline.
type RunPolicy int
// RunPolicy enumeration.
const (
RunOnSuccess RunPolicy = iota
RunOnFailure
RunAlways
RunNever
)
func (r RunPolicy) String() string {
return runPolicyID[r]
}
var runPolicyID = map[RunPolicy]string{
RunOnSuccess: "on-success",
RunOnFailure: "on-failure",
RunAlways: "always",
RunNever: "never",
}
var runPolicyName = map[string]RunPolicy{
"": RunOnSuccess,
"on-success": RunOnSuccess,
"on-failure": RunOnFailure,
"always": RunAlways,
"never": RunNever,
}
// MarshalJSON marshals the string representation of the
// run type to JSON.
func (r *RunPolicy) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString(`"`)
buffer.WriteString(runPolicyID[*r])
buffer.WriteString(`"`)
return buffer.Bytes(), nil
}
// UnmarshalJSON unmarshals the json representation of the
// run type from a string value.
func (r *RunPolicy) UnmarshalJSON(b []byte) error {
// unmarshal as string
var s string
err := json.Unmarshal(b, &s)
if err != nil {
return err
}
// lookup value
*r = runPolicyName[s]
return nil
}

115
engine/const_test.go
View File

@@ -10,121 +10,6 @@ import (
"testing"
)
//
// runtime policy unit tests.
//
func TestRunPolicy_Marshal(t *testing.T) {
tests := []struct {
policy RunPolicy
data string
}{
{
policy: RunAlways,
data: `"always"`,
},
{
policy: RunOnFailure,
data: `"on-failure"`,
},
{
policy: RunOnSuccess,
data: `"on-success"`,
},
{
policy: RunNever,
data: `"never"`,
},
}
for _, test := range tests {
data, err := json.Marshal(&test.policy)
if err != nil {
t.Error(err)
return
}
if bytes.Equal([]byte(test.data), data) == false {
t.Errorf("Failed to marshal policy %s", test.policy)
}
}
}
func TestRunPolicy_Unmarshal(t *testing.T) {
tests := []struct {
policy RunPolicy
data string
}{
{
policy: RunAlways,
data: `"always"`,
},
{
policy: RunOnFailure,
data: `"on-failure"`,
},
{
policy: RunOnSuccess,
data: `"on-success"`,
},
{
policy: RunNever,
data: `"never"`,
},
{
// no policy should default to on-success
policy: RunOnSuccess,
data: `""`,
},
}
for _, test := range tests {
var policy RunPolicy
err := json.Unmarshal([]byte(test.data), &policy)
if err != nil {
t.Error(err)
return
}
if got, want := policy, test.policy; got != want {
t.Errorf("Want policy %q, got %q", want, got)
}
}
}
func TestRunPolicy_UnmarshalTypeError(t *testing.T) {
var policy RunPolicy
err := json.Unmarshal([]byte("[]"), &policy)
if _, ok := err.(*json.UnmarshalTypeError); !ok {
t.Errorf("Expect unmarshal error return when JSON invalid")
}
}
func TestRunPolicy_String(t *testing.T) {
tests := []struct {
policy RunPolicy
value string
}{
{
policy: RunAlways,
value: "always",
},
{
policy: RunOnFailure,
value: "on-failure",
},
{
policy: RunOnSuccess,
value: "on-success",
},
}
for _, test := range tests {
if got, want := test.policy.String(), test.value; got != want {
t.Errorf("Want policy string %q, got %q", want, got)
}
}
}
//
// pull policy unit tests.
//
func TestPullPolicy_Marshal(t *testing.T) {
tests := []struct {
policy PullPolicy

293
engine/engine.go
View File

@@ -7,17 +7,288 @@ package engine
import (
"context"
"io"
"io/ioutil"
"github.com/drone-runners/drone-runner-docker/internal/docker/errors"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage"
"github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy"
"github.com/drone/runner-go/pipeline/runtime"
"github.com/drone/runner-go/registry/auths"
"docker.io/go-docker"
"docker.io/go-docker/api/types"
"docker.io/go-docker/api/types/network"
"docker.io/go-docker/api/types/volume"
)
// Engine is the interface that must be implemented by a
// pipeline execution engine.
type Engine interface {
// Setup the pipeline environment.
Setup(context.Context, *Spec) error
// Destroy the pipeline environment.
Destroy(context.Context, *Spec) error
// Run runs the pipeine step.
Run(context.Context, *Spec, *Step, io.Writer) (*State, error)
// Opts configures the Docker engine.
type Opts struct {
HidePull bool
}
// Docker implements a Docker pipeline engine.
type Docker struct {
client docker.APIClient
hidePull bool
}
// New returns a new engine.
func New(client docker.APIClient, opts Opts) *Docker {
return &Docker{
client: client,
hidePull: opts.HidePull,
}
}
// NewEnv returns a new Engine from the environment.
func NewEnv(opts Opts) (*Docker, error) {
cli, err := docker.NewEnvClient()
if err != nil {
return nil, err
}
return New(cli, opts), nil
}
// Ping pings the Docker daemon.
func (e *Docker) Ping(ctx context.Context) error {
_, err := e.client.Ping(ctx)
return err
}
// Setup the pipeline environment.
func (e *Docker) Setup(ctx context.Context, specv runtime.Spec) error {
spec := specv.(*Spec)
// creates the default temporary (local) volumes
// that are mounted into each container step.
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
_, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{
Name: vol.EmptyDir.ID,
Driver: "local",
Labels: vol.EmptyDir.Labels,
})
if err != nil {
return errors.TrimExtraInfo(err)
}
}
// creates the default pod network. All containers
// defined in the pipeline are attached to this network.
driver := "bridge"
if spec.Platform.OS == "windows" {
driver = "nat"
}
_, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{
Driver: driver,
Labels: spec.Network.Labels,
})
return errors.TrimExtraInfo(err)
}
// Destroy the pipeline environment.
func (e *Docker) Destroy(ctx context.Context, specv runtime.Spec) error {
spec := specv.(*Spec)
removeOpts := types.ContainerRemoveOptions{
Force: true,
RemoveLinks: false,
RemoveVolumes: true,
}
// stop all containers
for _, step := range spec.Steps {
e.client.ContainerKill(ctx, step.ID, "9")
}
// cleanup all containers
for _, step := range spec.Steps {
e.client.ContainerRemove(ctx, step.ID, removeOpts)
}
// cleanup all volumes
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
// tempfs volumes do not have a volume entry,
// and therefore do not require removal.
if vol.EmptyDir.Medium == "memory" {
continue
}
e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true)
}
// cleanup the network
e.client.NetworkRemove(ctx, spec.Network.ID)
// notice that we never collect or return any errors.
// this is because we silently ignore cleanup failures
// and instead ask the system admin to periodically run
// `docker prune` commands.
return nil
}
// Run runs the pipeline step.
func (e *Docker) Run(ctx context.Context, specv runtime.Spec, stepv runtime.Step, output io.Writer) (*runtime.State, error) {
spec := specv.(*Spec)
step := stepv.(*Step)
// create the container
err := e.create(ctx, spec, step, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// start the container
err = e.start(ctx, step.ID)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// tail the container
err = e.tail(ctx, step.ID, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// wait for the response
return e.wait(ctx, step.ID)
}
//
// emulate docker commands
//
func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error {
// create pull options with encoded authorization credentials.
pullopts := types.ImagePullOptions{}
if step.Auth != nil {
pullopts.RegistryAuth = auths.Header(
step.Auth.Username,
step.Auth.Password,
)
}
// automatically pull the latest version of the image if requested
// by the process configuration, or if the image is :latest
if step.Pull == PullAlways ||
(step.Pull == PullDefault && image.IsLatest(step.Image)) {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr == nil {
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
}
if pullerr != nil {
return pullerr
}
}
_, err := e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
// automatically pull and try to re-create the image if the
// failure is caused because the image does not exist.
if docker.IsErrImageNotFound(err) && step.Pull != PullNever {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr != nil {
return pullerr
}
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
// once the image is successfully pulled we attempt to
// re-create the container.
_, err = e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
}
if err != nil {
return err
}
// attach the container to user-defined networks.
// primarily used to attach global user-defined networks.
if step.Network == "" {
for _, net := range step.Networks {
err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{
Aliases: []string{net},
})
if err != nil {
return nil
}
}
}
return nil
}
// helper function emulates the `docker start` command.
func (e *Docker) start(ctx context.Context, id string) error {
return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{})
}
// helper function emulates the `docker wait` command, blocking
// until the container stops and returning the exit code.
func (e *Docker) wait(ctx context.Context, id string) (*runtime.State, error) {
wait, errc := e.client.ContainerWait(ctx, id, "")
select {
case <-wait:
case <-errc:
}
info, err := e.client.ContainerInspect(ctx, id)
if err != nil {
return nil, err
}
if info.State.Running {
// TODO(bradrydewski) if the state is still running
// we should call wait again.
}
return &runtime.State{
Exited: true,
ExitCode: info.State.ExitCode,
OOMKilled: info.State.OOMKilled,
}, nil
}
// helper function emulates the `docker logs -f` command, streaming
// all container logs until the container stops.
func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error {
opts := types.ContainerLogsOptions{
Follow: true,
ShowStdout: true,
ShowStderr: true,
Details: false,
Timestamps: false,
}
logs, err := e.client.ContainerLogs(ctx, id, opts)
if err != nil {
return err
}
go func() {
stdcopy.StdCopy(output, output, logs)
logs.Close()
}()
return nil
}

286
engine/engine_impl.go
View File

@@ -1,286 +0,0 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package engine
import (
"context"
"io"
"io/ioutil"
"github.com/drone-runners/drone-runner-docker/internal/docker/errors"
"github.com/drone-runners/drone-runner-docker/internal/docker/image"
"github.com/drone-runners/drone-runner-docker/internal/docker/jsonmessage"
"github.com/drone-runners/drone-runner-docker/internal/docker/stdcopy"
"github.com/drone/runner-go/registry/auths"
"docker.io/go-docker"
"docker.io/go-docker/api/types"
"docker.io/go-docker/api/types/network"
"docker.io/go-docker/api/types/volume"
)
// Opts configures the Docker engine.
type Opts struct {
HidePull bool
}
// Docker implements a Docker pipeline engine.
type Docker struct {
client docker.APIClient
hidePull bool
}
// New returns a new engine.
func New(client docker.APIClient, opts Opts) *Docker {
return &Docker{
client: client,
hidePull: opts.HidePull,
}
}
// NewEnv returns a new Engine from the environment.
func NewEnv(opts Opts) (*Docker, error) {
cli, err := docker.NewEnvClient()
if err != nil {
return nil, err
}
return New(cli, opts), nil
}
// Ping pings the Docker daemon.
func (e *Docker) Ping(ctx context.Context) error {
_, err := e.client.Ping(ctx)
return err
}
// Setup the pipeline environment.
func (e *Docker) Setup(ctx context.Context, spec *Spec) error {
// creates the default temporary (local) volumes
// that are mounted into each container step.
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
_, err := e.client.VolumeCreate(ctx, volume.VolumesCreateBody{
Name: vol.EmptyDir.ID,
Driver: "local",
Labels: vol.EmptyDir.Labels,
})
if err != nil {
return errors.TrimExtraInfo(err)
}
}
// creates the default pod network. All containers
// defined in the pipeline are attached to this network.
driver := "bridge"
if spec.Platform.OS == "windows" {
driver = "nat"
}
_, err := e.client.NetworkCreate(ctx, spec.Network.ID, types.NetworkCreate{
Driver: driver,
Labels: spec.Network.Labels,
})
return errors.TrimExtraInfo(err)
}
// Destroy the pipeline environment.
func (e *Docker) Destroy(ctx context.Context, spec *Spec) error {
removeOpts := types.ContainerRemoveOptions{
Force: true,
RemoveLinks: false,
RemoveVolumes: true,
}
// stop all containers
for _, step := range spec.Steps {
e.client.ContainerKill(ctx, step.ID, "9")
}
// cleanup all containers
for _, step := range spec.Steps {
e.client.ContainerRemove(ctx, step.ID, removeOpts)
}
// cleanup all volumes
for _, vol := range spec.Volumes {
if vol.EmptyDir == nil {
continue
}
// tempfs volumes do not have a volume entry,
// and therefore do not require removal.
if vol.EmptyDir.Medium == "memory" {
continue
}
e.client.VolumeRemove(ctx, vol.EmptyDir.ID, true)
}
// cleanup the network
e.client.NetworkRemove(ctx, spec.Network.ID)
// notice that we never collect or return any errors.
// this is because we silently ignore cleanup failures
// and instead ask the system admin to periodically run
// `docker prune` commands.
return nil
}
// Run runs the pipeline step.
func (e *Docker) Run(ctx context.Context, spec *Spec, step *Step, output io.Writer) (*State, error) {
// create the container
err := e.create(ctx, spec, step, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// start the container
err = e.start(ctx, step.ID)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// tail the container
err = e.tail(ctx, step.ID, output)
if err != nil {
return nil, errors.TrimExtraInfo(err)
}
// wait for the response
return e.wait(ctx, step.ID)
}
//
// emulate docker commands
//
func (e *Docker) create(ctx context.Context, spec *Spec, step *Step, output io.Writer) error {
// create pull options with encoded authorization credentials.
pullopts := types.ImagePullOptions{}
if step.Auth != nil {
pullopts.RegistryAuth = auths.Header(
step.Auth.Username,
step.Auth.Password,
)
}
// automatically pull the latest version of the image if requested
// by the process configuration, or if the image is :latest
if step.Pull == PullAlways ||
(step.Pull == PullDefault && image.IsLatest(step.Image)) {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr == nil {
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
}
if pullerr != nil {
return pullerr
}
}
_, err := e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
// automatically pull and try to re-create the image if the
// failure is caused because the image does not exist.
if docker.IsErrImageNotFound(err) && step.Pull != PullNever {
rc, pullerr := e.client.ImagePull(ctx, step.Image, pullopts)
if pullerr != nil {
return pullerr
}
if e.hidePull {
io.Copy(ioutil.Discard, rc)
} else {
jsonmessage.Copy(rc, output)
}
rc.Close()
// once the image is successfully pulled we attempt to
// re-create the container.
_, err = e.client.ContainerCreate(ctx,
toConfig(spec, step),
toHostConfig(spec, step),
toNetConfig(spec, step),
step.ID,
)
}
if err != nil {
return err
}
// attach the container to user-defined networks.
// primarily used to attach global user-defined networks.
if step.Network == "" {
for _, net := range step.Networks {
err = e.client.NetworkConnect(ctx, net, step.ID, &network.EndpointSettings{
Aliases: []string{net},
})
if err != nil {
return nil
}
}
}
return nil
}
// helper function emulates the `docker start` command.
func (e *Docker) start(ctx context.Context, id string) error {
return e.client.ContainerStart(ctx, id, types.ContainerStartOptions{})
}
// helper function emulates the `docker wait` command, blocking
// until the container stops and returning the exit code.
func (e *Docker) wait(ctx context.Context, id string) (*State, error) {
wait, errc := e.client.ContainerWait(ctx, id, "")
select {
case <-wait:
case <-errc:
}
info, err := e.client.ContainerInspect(ctx, id)
if err != nil {
return nil, err
}
if info.State.Running {
// TODO(bradrydewski) if the state is still running
// we should call wait again.
}
return &State{
Exited: true,
ExitCode: info.State.ExitCode,
OOMKilled: info.State.OOMKilled,
}, nil
}
// helper function emulates the `docker logs -f` command, streaming
// all container logs until the container stops.
func (e *Docker) tail(ctx context.Context, id string, output io.Writer) error {
opts := types.ContainerLogsOptions{
Follow: true,
ShowStdout: true,
ShowStderr: true,
Details: false,
Timestamps: false,
}
logs, err := e.client.ContainerLogs(ctx, id, opts)
if err != nil {
return err
}
go func() {
stdcopy.StdCopy(output, output, logs)
logs.Close()
}()
return nil
}

6
engine/linter/linter.go
View File

@@ -11,6 +11,8 @@ import (
"strings"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/manifest"
)
// ErrDuplicateStepName is returned when two Pipeline steps
@@ -43,8 +45,8 @@ func New() *Linter {
// Lint executes the linting rules for the pipeline
// configuration.
func (l *Linter) Lint(pipeline *resource.Pipeline, opts Opts) error {
return checkPipeline(pipeline, opts.Trusted)
func (l *Linter) Lint(pipeline manifest.Resource, repo *drone.Repo) error {
return checkPipeline(pipeline.(*resource.Pipeline), repo.Trusted)
}
func checkPipeline(pipeline *resource.Pipeline, trusted bool) error {

3
engine/linter/linter_test.go
View File

@@ -9,6 +9,7 @@ import (
"testing"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/drone-go/drone"
"github.com/drone/runner-go/manifest"
)
@@ -212,7 +213,7 @@ func TestLint(t *testing.T) {
}
lint := New()
opts := Opts{Trusted: test.trusted}
opts := &drone.Repo{Trusted: test.trusted}
err = lint.Lint(resources.Resources[0].(*resource.Pipeline), opts)
if err == nil && test.invalid == true {
t.Logf("yaml: %s", test.path)

56
engine/replacer/replacer.go
View File

@@ -1,56 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package replacer
import (
"fmt"
"io"
"strings"
"github.com/drone-runners/drone-runner-docker/engine"
)
const maskedf = "[secret:%s]"
// Replacer is an io.Writer that finds and masks sensitive data.
type Replacer struct {
w io.WriteCloser
r *strings.Replacer
}
// New returns a replacer that wraps writer w.
func New(w io.WriteCloser, secrets []*engine.Secret) io.WriteCloser {
var oldnew []string
for _, secret := range secrets {
if len(secret.Data) == 0 || secret.Mask == false {
continue
}
name := strings.ToLower(secret.Name)
masked := fmt.Sprintf(maskedf, name)
oldnew = append(oldnew, string(secret.Data))
oldnew = append(oldnew, masked)
}
if len(oldnew) == 0 {
return w
}
return &Replacer{
w: w,
r: strings.NewReplacer(oldnew...),
}
}
// Write writes p to the base writer. The method scans for any
// sensitive data in p and masks before writing.
func (r *Replacer) Write(p []byte) (n int, err error) {
_, err = r.w.Write([]byte(r.r.Replace(string(p))))
return len(p), err
}
// Close closes the base writer.
func (r *Replacer) Close() error {
return r.w.Close()
}

56
engine/replacer/replacer_test.go
View File

@@ -1,56 +0,0 @@
// Code generated automatically. DO NOT EDIT.
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package replacer
import (
"bytes"
"io"
"testing"
"github.com/drone-runners/drone-runner-docker/engine"
)
func TestReplace(t *testing.T) {
secrets := []*engine.Secret{
{Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false},
{Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: true},
{Name: "DOCKER_EMAIL", Data: []byte(""), Mask: true},
}
buf := new(bytes.Buffer)
w := New(&nopCloser{buf}, secrets)
w.Write([]byte("username octocat password correct-horse-batter-staple"))
w.Close()
if got, want := buf.String(), "username octocat password [secret:docker_password]"; got != want {
t.Errorf("Want masked string %s, got %s", want, got)
}
}
// this test verifies that if there are no secrets to scan and
// mask, the io.WriteCloser is returned as-is.
func TestReplaceNone(t *testing.T) {
secrets := []*engine.Secret{
{Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false},
{Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: false},
}
buf := new(bytes.Buffer)
w := &nopCloser{buf}
r := New(w, secrets)
if w != r {
t.Errorf("Expect buffer returned with no replacer")
}
}
type nopCloser struct {
io.Writer
}
func (*nopCloser) Close() error {
return nil
}

2
engine/resource/lookup.go
View File

@@ -11,7 +11,7 @@ import (
)
// Lookup returns the named pipeline from the Manifest.
func Lookup(name string, manifest *manifest.Manifest) (*Pipeline, error) {
func Lookup(name string, manifest *manifest.Manifest) (manifest.Resource, error) {
for _, resource := range manifest.Resources {
if !isNameMatch(resource.GetName(), name) {
continue

62
engine/spec.go
View File

@@ -4,7 +4,13 @@
package engine
import (
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/pipeline/runtime"
)
type (
// Spec provides the pipeline spec. This provides the
// required instructions for reproducible pipeline
// execution.
@@ -44,7 +50,7 @@ type (
Networks []string `json:"networks,omitempty"`
Privileged bool `json:"privileged,omitempty"`
Pull PullPolicy `json:"pull,omitempty"`
RunPolicy RunPolicy `json:"run_policy,omitempty"`
RunPolicy runtime.RunPolicy `json:"run_policy,omitempty"`
Secrets []*Secret `json:"secrets,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
User string `json:"user,omitempty"`
@@ -52,14 +58,6 @@ type (
WorkingDir string `json:"working_dir,omitempty"`
}
// Platform defines the target platform.
Platform struct {
OS string `json:"os,omitempty"`
Arch string `json:"arch,omitempty"`
Variant string `json:"variant,omitempty"`
Version string `json:"version,omitempty"`
}
// Secret represents a secret variable.
Secret struct {
Name string `json:"name,omitempty"`
@@ -68,11 +66,12 @@ type (
Mask bool `json:"mask,omitempty"`
}
// State represents the process state.
State struct {
ExitCode int // Container exit code
Exited bool // Container exited
OOMKilled bool // Container is oom killed
// Platform defines the target platform.
Platform struct {
OS string `json:"os,omitempty"`
Arch string `json:"arch,omitempty"`
Variant string `json:"variant,omitempty"`
Version string `json:"version,omitempty"`
}
// Volume that can be mounted by containers.
@@ -128,3 +127,38 @@ type (
Password string `json:"password,omitempty"`
}
)
//
// implements the Spec interface
//
func (s *Spec) StepLen() int { return len(s.Steps) }
func (s *Spec) StepAt(i int) runtime.Step { return s.Steps[i] }
//
// implements the Secret interface
//
func (s *Secret) GetName() string { return s.Name }
func (s *Secret) GetValue() string { return string(s.Data) }
func (s *Secret) IsMasked() bool { return s.Mask }
//
// implements the Step interface
//
func (s *Step) GetName() string { return s.Name }
func (s *Step) GetDependencies() []string { return s.DependsOn }
func (s *Step) GetEnviron() map[string]string { return s.Envs }
func (s *Step) SetEnviron(env map[string]string) { s.Envs = env }
func (s *Step) GetErrPolicy() runtime.ErrPolicy { return runtime.ErrFail }
func (s *Step) GetRunPolicy() runtime.RunPolicy { return s.RunPolicy }
func (s *Step) GetSecretAt(i int) runtime.Secret { return s.Secrets[i] }
func (s *Step) GetSecretLen() int { return len(s.Secrets) }
func (s *Step) IsDetached() bool { return s.Detach }
func (s *Step) Clone() runtime.Step {
dst := new(Step)
*dst = *s
dst.Envs = environ.Combine(s.Envs)
return dst
}