diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..12b295a --- /dev/null +++ b/.drone.yml @@ -0,0 +1,48 @@ +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test + image: golang:1.12 + commands: + - go test -cover ./... + volumes: + - name: go + path: /go + +- name: build + image: golang:1.12 + commands: + - sh scripts/build.sh + volumes: + - name: go + path: /go + when: + event: + - push + - tag + +- name: publish + image: plugins/docker + pull: if-not-exists + settings: + repo: drone/drone-runner-docker + auto_tag: true + auto_tag_suffix: linux-amd64 + dockerfile: docker/Dockerfile.linux.amd64 + username: + from_secret: docker_username + password: + from_secret: docker_password + when: + ref: + - refs/heads/master + - refs/tags/* + +volumes: +- name: go + temp: {} diff --git a/.github/code_of_conduct.md b/.github/code_of_conduct.md new file mode 100644 index 0000000..7134070 --- /dev/null +++ b/.github/code_of_conduct.md @@ -0,0 +1,73 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, +religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at conduct@drone.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..e69de29 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..e69de29 diff --git a/.github/security.md b/.github/security.md new file mode 100644 index 0000000..e80b618 --- /dev/null +++ b/.github/security.md @@ -0,0 +1,36 @@ +# Security Policies and Procedures + +This document outlines security procedures and general policies for this project. + + * [Reporting a Bug](#reporting-a-bug) + * [Disclosure Policy](#disclosure-policy) + * [Comments on this Policy](#comments-on-this-policy) + +## Reporting a Bug + +Report security bugs by emailing the lead maintainer at security@drone.io. + +The lead maintainer will acknowledge your email within 48 hours, and will send a +more detailed response within 48 hours indicating the next steps in handling +your report. After the initial reply to your report, the security team will +endeavor to keep you informed of the progress towards a fix and full +announcement, and may ask for additional information or guidance. + +Report security bugs in third-party packages to the person or team maintaining +the module. + +## Disclosure Policy + +When the security team receives a security bug report, they will assign it to a +primary handler. This person will coordinate the fix and release process, +involving the following steps: + + * Confirm the problem and determine the affected versions. + * Audit code to find any potential similar problems. + * Prepare fixes for all releases still under maintenance. These fixes will be + released as fast as possible. + +## Comments on this Policy + +If you have suggestions on how this process could be improved please submit a +pull request. \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c49ffca --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +drone-runner-docker +drone-runner-docker.exe +release/* +.env +NOTES* +engine/compiler2 diff --git a/BUILDING.md b/BUILDING.md new file mode 100644 index 0000000..2ec7444 --- /dev/null +++ b/BUILDING.md @@ -0,0 +1,14 @@ +1. Install go 1.13 or higher +2. Test + + go test ./... + +3. Build binaries + + sh scripts/build_all.sh + +4. Build images + + docker build -t drone/drone-runner-docker:latest-linux-amd64 -f docker/Dockerfile.linux.amd64 . + docker build -t drone/drone-runner-docker:latest-linux-arm64 -f docker/Dockerfile.linux.arm64 . + docker build -t drone/drone-runner-docker:latest-linux-arm -f docker/Dockerfile.linux.arm . diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..69b0890 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Added diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..2f4f432 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,2 @@ +[Polyform-Small-Business-1.0.0](https://polyformproject.org/licenses/small-business/1.0.0) OR +[Polyform-Free-Trial-1.0.0](https://polyformproject.org/licenses/free-trial/1.0.0) \ No newline at end of file diff --git a/README.md b/README.md index d87040d..bffe523 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,12 @@ # drone-runner-docker -Drone pipeline runner that executes builds inside Docker containers + +The `docker` runner executes pipelines inside Docker containers. This runner is intended for linux workloads that are suitable for execution inside containers. This requires Drone server `1.6.0` or higher. + +Documentation:
+https://docker-runner.docs.drone.io + +Technical Support:
+https://discourse.drone.io + +Issue Tracker and Roadmap:
+https://trello.com/b/ttae5E5o/drone diff --git a/command/command.go b/command/command.go new file mode 100644 index 0000000..afddd32 --- /dev/null +++ b/command/command.go @@ -0,0 +1,32 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package command + +import ( + "context" + "os" + + "github.com/drone-runners/drone-runner-docker/command/daemon" + + "gopkg.in/alecthomas/kingpin.v2" +) + +// program version +var version = "0.0.0" + +// empty context +var nocontext = context.Background() + +// Command parses the command line arguments and then executes a +// subcommand program. +func Command() { + app := kingpin.New("drone", "drone exec runner") + registerCompile(app) + registerExec(app) + daemon.Register(app) + + kingpin.Version(version) + kingpin.MustParse(app.Parse(os.Args[1:])) +} diff --git a/command/compile.go b/command/compile.go new file mode 100644 index 0000000..7993df9 --- /dev/null +++ b/command/compile.go @@ -0,0 +1,122 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package command + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/drone-runners/drone-runner-docker/command/internal" + "github.com/drone-runners/drone-runner-docker/engine/compiler" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/envsubst" + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/secret" + + "gopkg.in/alecthomas/kingpin.v2" +) + +type compileCommand struct { + *internal.Flags + + Source *os.File + Environ map[string]string + Secrets map[string]string +} + +func (c *compileCommand) run(*kingpin.ParseContext) error { + rawsource, err := ioutil.ReadAll(c.Source) + if err != nil { + return err + } + + envs := environ.Combine( + c.Environ, + environ.System(c.System), + environ.Repo(c.Repo), + environ.Build(c.Build), + environ.Stage(c.Stage), + environ.Link(c.Repo, c.Build, c.System), + c.Build.Params, + ) + + // string substitution function ensures that string + // replacement variables are escaped and quoted if they + // contain newlines. + subf := func(k string) string { + v := envs[k] + if strings.Contains(v, "\n") { + v = fmt.Sprintf("%q", v) + } + return v + } + + // evaluates string replacement expressions and returns an + // update configuration. + config, err := envsubst.Eval(string(rawsource), subf) + if err != nil { + return err + } + + // parse and lint the configuration + manifest, err := manifest.ParseString(config) + if err != nil { + return err + } + + // a configuration can contain multiple pipelines. + // get a specific pipeline resource for execution. + resource, err := resource.Lookup(c.Stage.Name, manifest) + if err != nil { + return err + } + + // compile the pipeline to an intermediate representation. + comp := &compiler.Compiler{ + Pipeline: resource, + Manifest: manifest, + Build: c.Build, + Netrc: c.Netrc, + Repo: c.Repo, + Stage: c.Stage, + System: c.System, + Environ: c.Environ, + Secret: secret.StaticVars(c.Secrets), + } + spec := comp.Compile(nocontext) + + // encode the pipeline in json format and print to the + // console for inspection. + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(spec) + return nil +} + +func registerCompile(app *kingpin.Application) { + c := new(compileCommand) + c.Environ = map[string]string{} + c.Secrets = map[string]string{} + + cmd := app.Command("compile", "compile the yaml file"). + Action(c.run) + + cmd.Flag("source", "source file location"). + Default(".drone.yml"). + FileVar(&c.Source) + + cmd.Flag("secrets", "secret parameters"). + StringMapVar(&c.Secrets) + + cmd.Flag("environ", "environment variables"). + StringMapVar(&c.Environ) + + // shared pipeline flags + c.Flags = internal.ParseFlags(cmd) +} diff --git a/command/daemon/config.go b/command/daemon/config.go new file mode 100644 index 0000000..f2d2017 --- /dev/null +++ b/command/daemon/config.go @@ -0,0 +1,94 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package daemon + +import ( + "fmt" + "os" + + "github.com/kelseyhightower/envconfig" +) + +// Config stores the system configuration. +type Config struct { + Debug bool `envconfig:"DRONE_DEBUG"` + Trace bool `envconfig:"DRONE_TRACE"` + + Logger struct { + File string `envconfig:"DRONE_LOG_FILE"` + MaxAge int `envconfig:"DRONE_LOG_FILE_MAX_AGE" default:"1"` + MaxBackups int `envconfig:"DRONE_LOG_FILE_MAX_BACKUPS" default:"1"` + MaxSize int `envconfig:"DRONE_LOG_FILE_MAX_SIZE" default:"100"` + } + + Client struct { + Address string `ignored:"true"` + Proto string `envconfig:"DRONE_RPC_PROTO" default:"http"` + Host string `envconfig:"DRONE_RPC_HOST" required:"true"` + Secret string `envconfig:"DRONE_RPC_SECRET" required:"true"` + SkipVerify bool `envconfig:"DRONE_RPC_SKIP_VERIFY"` + Dump bool `envconfig:"DRONE_RPC_DUMP_HTTP"` + DumpBody bool `envconfig:"DRONE_RPC_DUMP_HTTP_BODY"` + } + + Dashboard struct { + Disabled bool `envconfig:"DRONE_UI_DISABLE"` + Username string `envconfig:"DRONE_UI_USERNAME"` + Password string `envconfig:"DRONE_UI_PASSWORD"` + Realm string `envconfig:"DRONE_UI_REALM" default:"MyRealm"` + } + + Server struct { + Proto string `envconfig:"DRONE_SERVER_PROTO"` + Host string `envconfig:"DRONE_SERVER_HOST"` + Port string `envconfig:"DRONE_SERVER_PORT" default:":3000"` + Acme bool `envconfig:"DRONE_SERVER_ACME"` + } + + Keypair struct { + Public string `envconfig:"DRONE_PUBLIC_KEY_FILE"` + Private string `envconfig:"DRONE_PRIVATE_KEY_FILE"` + } + + Runner struct { + Name string `envconfig:"DRONE_RUNNER_NAME"` + Capacity int `envconfig:"DRONE_RUNNER_CAPACITY" default:"10"` + Procs int64 `envconfig:"DRONE_RUNNER_MAX_PROCS"` + Labels map[string]string `envconfig:"DRONE_RUNNER_LABELS"` + Environ map[string]string `envconfig:"DRONE_RUNNER_ENVIRON"` + } + + Limit struct { + Repos []string `envconfig:"DRONE_LIMIT_REPOS"` + Events []string `envconfig:"DRONE_LIMIT_EVENTS"` + Trusted bool `envconfig:"DRONE_LIMIT_TRUSTED"` + } + + Secret struct { + Endpoint string `envconfig:"DRONE_SECRET_PLUGIN_ENDPOINT"` + Token string `envconfig:"DRONE_SECRET_PLUGIN_TOKEN"` + SkipVerify bool `envconfig:"DRONE_SECRET_PLUGIN_SKIP_VERIFY"` + } +} + +func fromEnviron() (Config, error) { + var config Config + err := envconfig.Process("", &config) + if err != nil { + return config, err + } + if config.Runner.Name == "" { + config.Runner.Name, _ = os.Hostname() + } + if config.Dashboard.Password == "" { + config.Dashboard.Disabled = true + } + config.Client.Address = fmt.Sprintf( + "%s://%s", + config.Client.Proto, + config.Client.Host, + ) + return config, nil +} diff --git a/command/daemon/daemon.go b/command/daemon/daemon.go new file mode 100644 index 0000000..2f33279 --- /dev/null +++ b/command/daemon/daemon.go @@ -0,0 +1,204 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package daemon + +import ( + "context" + "time" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone-runners/drone-runner-docker/internal/match" + "github.com/drone-runners/drone-runner-docker/runtime" + + "github.com/drone/runner-go/client" + "github.com/drone/runner-go/handler/router" + "github.com/drone/runner-go/logger" + loghistory "github.com/drone/runner-go/logger/history" + "github.com/drone/runner-go/pipeline/history" + "github.com/drone/runner-go/pipeline/remote" + "github.com/drone/runner-go/secret" + "github.com/drone/runner-go/server" + "github.com/drone/signal" + + "github.com/joho/godotenv" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + "gopkg.in/alecthomas/kingpin.v2" +) + +// empty context. +var nocontext = context.Background() + +type daemonCommand struct { + envfile string +} + +func (c *daemonCommand) run(*kingpin.ParseContext) error { + // load environment variables from file. + godotenv.Load(c.envfile) + + // load the configuration from the environment + config, err := fromEnviron() + if err != nil { + return err + } + + // setup the global logrus logger. + setupLogger(config) + + cli := client.New( + config.Client.Address, + config.Client.Secret, + config.Client.SkipVerify, + ) + if config.Client.Dump { + cli.Dumper = logger.StandardDumper( + config.Client.DumpBody, + ) + } + cli.Logger = logger.Logrus( + logrus.NewEntry( + logrus.StandardLogger(), + ), + ) + + engine, err := engine.New(config.Keypair.Public, config.Keypair.Private) + if err != nil { + return err + } + + remote := remote.New(cli) + tracer := history.New(remote) + hook := loghistory.New() + logrus.AddHook(hook) + + poller := &runtime.Poller{ + Client: cli, + Runner: &runtime.Runner{ + Client: cli, + Environ: config.Runner.Environ, + Machine: config.Runner.Name, + Reporter: tracer, + Match: match.Func( + config.Limit.Repos, + config.Limit.Events, + config.Limit.Trusted, + ), + Secret: secret.External( + config.Secret.Endpoint, + config.Secret.Token, + config.Secret.SkipVerify, + ), + Execer: runtime.NewExecer( + tracer, + remote, + engine, + config.Runner.Procs, + ), + }, + Filter: &client.Filter{ + Kind: resource.Kind, + Type: resource.Type, + Labels: config.Runner.Labels, + }, + } + + ctx, cancel := context.WithCancel(nocontext) + defer cancel() + + // listen for termination signals to gracefully shutdown + // the runner daemon. + ctx = signal.WithContextFunc(ctx, func() { + println("received signal, terminating process") + cancel() + }) + + var g errgroup.Group + server := server.Server{ + Addr: config.Server.Port, + Handler: router.New(tracer, hook, router.Config{ + Username: config.Dashboard.Username, + Password: config.Dashboard.Password, + Realm: config.Dashboard.Realm, + }), + } + + logrus.WithField("addr", config.Server.Port). + Infoln("starting the server") + + g.Go(func() error { + return server.ListenAndServe(ctx) + }) + + // Ping the server and block until a successful connection + // to the server has been established. + for { + err := cli.Ping(ctx, config.Runner.Name) + select { + case <-ctx.Done(): + return nil + default: + } + if ctx.Err() != nil { + break + } + if err != nil { + logrus.WithError(err). + Errorln("cannot ping the remote server") + time.Sleep(time.Second) + } else { + logrus.Infoln("successfully pinged the remote server") + break + } + } + + g.Go(func() error { + logrus.WithField("capacity", config.Runner.Capacity). + WithField("endpoint", config.Client.Address). + WithField("kind", resource.Kind). + WithField("type", resource.Type). + Infoln("polling the remote server") + + poller.Poll(ctx, config.Runner.Capacity) + return nil + }) + + err = g.Wait() + if err != nil { + logrus.WithError(err). + Errorln("shutting down the server") + } + return err +} + +// helper function configures the global logger from +// the loaded configuration. +func setupLogger(config Config) { + logger.Default = logger.Logrus( + logrus.NewEntry( + logrus.StandardLogger(), + ), + ) + if config.Debug { + logrus.SetLevel(logrus.DebugLevel) + } + if config.Trace { + logrus.SetLevel(logrus.TraceLevel) + } +} + +// Register the daemon command. +func Register(app *kingpin.Application) { + c := new(daemonCommand) + + cmd := app.Command("daemon", "starts the runner daemon"). + Default(). + Action(c.run) + + cmd.Arg("envfile", "load the environment variable file"). + Default(""). + StringVar(&c.envfile) +} diff --git a/command/exec.go b/command/exec.go new file mode 100644 index 0000000..724961d --- /dev/null +++ b/command/exec.go @@ -0,0 +1,234 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package command + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + "time" + + "github.com/drone-runners/drone-runner-docker/command/internal" + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/compiler" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone-runners/drone-runner-docker/runtime" + "github.com/drone/drone-go/drone" + "github.com/drone/envsubst" + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/logger" + "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline" + "github.com/drone/runner-go/pipeline/console" + "github.com/drone/runner-go/secret" + "github.com/drone/signal" + + "github.com/mattn/go-isatty" + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" +) + +type execCommand struct { + *internal.Flags + + Source *os.File + Environ map[string]string + Secrets map[string]string + Pretty bool + Procs int64 + Debug bool + Trace bool + Dump bool + PublicKey string + PrivateKey string +} + +func (c *execCommand) run(*kingpin.ParseContext) error { + rawsource, err := ioutil.ReadAll(c.Source) + if err != nil { + return err + } + + envs := environ.Combine( + c.Environ, + environ.System(c.System), + environ.Repo(c.Repo), + environ.Build(c.Build), + environ.Stage(c.Stage), + environ.Link(c.Repo, c.Build, c.System), + c.Build.Params, + ) + + // string substitution function ensures that string + // replacement variables are escaped and quoted if they + // contain newlines. + subf := func(k string) string { + v := envs[k] + if strings.Contains(v, "\n") { + v = fmt.Sprintf("%q", v) + } + return v + } + + // evaluates string replacement expressions and returns an + // update configuration. + config, err := envsubst.Eval(string(rawsource), subf) + if err != nil { + return err + } + + // parse and lint the configuration. + manifest, err := manifest.ParseString(config) + if err != nil { + return err + } + + // a configuration can contain multiple pipelines. + // get a specific pipeline resource for execution. + resource, err := resource.Lookup(c.Stage.Name, manifest) + if err != nil { + return err + } + + // compile the pipeline to an intermediate representation. + comp := &compiler.Compiler{ + Pipeline: resource, + Manifest: manifest, + Build: c.Build, + Netrc: c.Netrc, + Repo: c.Repo, + Stage: c.Stage, + System: c.System, + Environ: c.Environ, + Secret: secret.StaticVars(c.Secrets), + } + spec := comp.Compile(nocontext) + + // create a step object for each pipeline step. + for _, step := range spec.Steps { + if step.RunPolicy == engine.RunNever { + continue + } + c.Stage.Steps = append(c.Stage.Steps, &drone.Step{ + StageID: c.Stage.ID, + Number: len(c.Stage.Steps) + 1, + Name: step.Name, + Status: drone.StatusPending, + ErrIgnore: step.IgnoreErr, + }) + } + + // configures the pipeline timeout. + timeout := time.Duration(c.Repo.Timeout) * time.Minute + ctx, cancel := context.WithTimeout(nocontext, timeout) + defer cancel() + + // listen for operating system signals and cancel execution + // when received. + ctx = signal.WithContextFunc(ctx, func() { + println("received signal, terminating process") + cancel() + }) + + state := &pipeline.State{ + Build: c.Build, + Stage: c.Stage, + Repo: c.Repo, + System: c.System, + } + + // enable debug logging + logrus.SetLevel(logrus.WarnLevel) + if c.Debug { + logrus.SetLevel(logrus.DebugLevel) + } + if c.Trace { + logrus.SetLevel(logrus.TraceLevel) + } + logger.Default = logger.Logrus( + logrus.NewEntry( + logrus.StandardLogger(), + ), + ) + + engine, err := engine.New(c.PublicKey, c.PrivateKey) + if err != nil { + return err + } + + err = runtime.NewExecer( + pipeline.NopReporter(), + console.New(c.Pretty), + engine, + c.Procs, + ).Exec(ctx, spec, state) + + if c.Dump { + dump(state) + } + if err != nil { + return err + } + switch state.Stage.Status { + case drone.StatusError, drone.StatusFailing, drone.StatusKilled: + os.Exit(1) + } + return nil +} + +func dump(v interface{}) { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(v) +} + +func registerExec(app *kingpin.Application) { + c := new(execCommand) + c.Environ = map[string]string{} + c.Secrets = map[string]string{} + + cmd := app.Command("exec", "executes a pipeline"). + Action(c.run) + + cmd.Arg("source", "source file location"). + Default(".drone.yml"). + FileVar(&c.Source) + + cmd.Flag("secrets", "secret parameters"). + StringMapVar(&c.Secrets) + + cmd.Flag("environ", "environment variables"). + StringMapVar(&c.Environ) + + cmd.Flag("public-key", "public key file path"). + ExistingFileVar(&c.PublicKey) + + cmd.Flag("private-key", "private key file path"). + ExistingFileVar(&c.PrivateKey) + + cmd.Flag("debug", "enable debug logging"). + BoolVar(&c.Debug) + + cmd.Flag("trace", "enable trace logging"). + BoolVar(&c.Trace) + + cmd.Flag("dump", "dump the pipeline state to stdout"). + BoolVar(&c.Dump) + + cmd.Flag("pretty", "pretty print the output"). + Default( + fmt.Sprint( + isatty.IsTerminal( + os.Stdout.Fd(), + ), + ), + ).BoolVar(&c.Pretty) + + // shared pipeline flags + c.Flags = internal.ParseFlags(cmd) +} diff --git a/command/internal/flags.go b/command/internal/flags.go new file mode 100644 index 0000000..ab9c2b2 --- /dev/null +++ b/command/internal/flags.go @@ -0,0 +1,103 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package internal + +import ( + "fmt" + "time" + + "github.com/drone/drone-go/drone" + + "gopkg.in/alecthomas/kingpin.v2" +) + +// Flags maps +type Flags struct { + Build *drone.Build + Netrc *drone.Netrc + Repo *drone.Repo + Stage *drone.Stage + System *drone.System +} + +// ParseFlags parses the flags from the command args. +func ParseFlags(cmd *kingpin.CmdClause) *Flags { + f := &Flags{ + Build: &drone.Build{}, + Netrc: &drone.Netrc{}, + Repo: &drone.Repo{}, + Stage: &drone.Stage{}, + System: &drone.System{}, + } + + now := fmt.Sprint( + time.Now().Unix(), + ) + + cmd.Flag("repo-id", "repo id").Default("1").Int64Var(&f.Repo.ID) + cmd.Flag("repo-namespace", "repo namespace").Default("").StringVar(&f.Repo.Namespace) + cmd.Flag("repo-name", "repo name").Default("").StringVar(&f.Repo.Name) + cmd.Flag("repo-slug", "repo slug").Default("").StringVar(&f.Repo.Slug) + cmd.Flag("repo-http", "repo http clone url").Default("").StringVar(&f.Repo.HTTPURL) + cmd.Flag("repo-ssh", "repo ssh clone url").Default("").StringVar(&f.Repo.SSHURL) + cmd.Flag("repo-link", "repo link").Default("").StringVar(&f.Repo.Link) + cmd.Flag("repo-branch", "repo branch").Default("").StringVar(&f.Repo.Branch) + cmd.Flag("repo-private", "repo private").Default("false").BoolVar(&f.Repo.Private) + cmd.Flag("repo-visibility", "repo visibility").Default("").StringVar(&f.Repo.Visibility) + cmd.Flag("repo-trusted", "repo trusted").Default("false").BoolVar(&f.Repo.Trusted) + cmd.Flag("repo-protected", "repo protected").Default("false").BoolVar(&f.Repo.Protected) + cmd.Flag("repo-timeout", "repo timeout in minutes").Default("60").Int64Var(&f.Repo.Timeout) + cmd.Flag("repo-created", "repo created").Default(now).Int64Var(&f.Repo.Created) + cmd.Flag("repo-updated", "repo updated").Default(now).Int64Var(&f.Repo.Updated) + + cmd.Flag("build-id", "build id").Default("1").Int64Var(&f.Build.ID) + cmd.Flag("build-number", "build number").Default("1").Int64Var(&f.Build.Number) + cmd.Flag("build-parent", "build parent").Default("0").Int64Var(&f.Build.Parent) + cmd.Flag("build-event", "build event").Default("push").StringVar(&f.Build.Event) + cmd.Flag("build-action", "build action").Default("").StringVar(&f.Build.Action) + cmd.Flag("build-cron", "build cron trigger").Default("").StringVar(&f.Build.Cron) + cmd.Flag("build-target", "build deploy target").Default("").StringVar(&f.Build.Deploy) + cmd.Flag("build-created", "build created").Default(now).Int64Var(&f.Build.Created) + cmd.Flag("build-updated", "build updated").Default(now).Int64Var(&f.Build.Updated) + + cmd.Flag("commit-sender", "commit sender").Default("").StringVar(&f.Build.Sender) + cmd.Flag("commit-link", "commit link").Default("").StringVar(&f.Build.Link) + cmd.Flag("commit-title", "commit title").Default("").StringVar(&f.Build.Title) + cmd.Flag("commit-message", "commit message").Default("").StringVar(&f.Build.Message) + cmd.Flag("commit-before", "commit before").Default("").StringVar(&f.Build.Before) + cmd.Flag("commit-after", "commit after").Default("").StringVar(&f.Build.After) + cmd.Flag("commit-ref", "commit ref").Default("").StringVar(&f.Build.Ref) + cmd.Flag("commit-fork", "commit fork").Default("").StringVar(&f.Build.Fork) + cmd.Flag("commit-source", "commit source branch").Default("").StringVar(&f.Build.Source) + cmd.Flag("commit-target", "commit target branch").Default("").StringVar(&f.Build.Target) + + cmd.Flag("author-login", "commit author login").Default("").StringVar(&f.Build.Author) + cmd.Flag("author-name", "commit author name").Default("").StringVar(&f.Build.AuthorName) + cmd.Flag("author-email", "commit author email").Default("").StringVar(&f.Build.AuthorEmail) + cmd.Flag("author-avatar", "commit author avatar").Default("").StringVar(&f.Build.AuthorAvatar) + + cmd.Flag("stage-id", "stage id").Default("1").Int64Var(&f.Stage.ID) + cmd.Flag("stage-number", "stage number").Default("1").IntVar(&f.Stage.Number) + cmd.Flag("stage-kind", "stage kind").Default("").StringVar(&f.Stage.Kind) + cmd.Flag("stage-type", "stage type").Default("").StringVar(&f.Stage.Type) + cmd.Flag("stage-name", "stage name").Default("default").StringVar(&f.Stage.Name) + cmd.Flag("stage-os", "stage os").Default("").StringVar(&f.Stage.OS) + cmd.Flag("stage-arch", "stage arch").Default("").StringVar(&f.Stage.Arch) + cmd.Flag("stage-variant", "stage variant").Default("").StringVar(&f.Stage.Variant) + cmd.Flag("stage-kernel", "stage kernel").Default("").StringVar(&f.Stage.Kernel) + cmd.Flag("stage-created", "stage created").Default(now).Int64Var(&f.Stage.Created) + cmd.Flag("stage-updated", "stage updated").Default(now).Int64Var(&f.Stage.Updated) + + cmd.Flag("netrc-username", "netrc username").Default("").StringVar(&f.Netrc.Login) + cmd.Flag("netrc-password", "netrc password").Default("").StringVar(&f.Netrc.Password) + cmd.Flag("netrc-machine", "netrc machine").Default("").StringVar(&f.Netrc.Machine) + + cmd.Flag("system-host", "server host").Default("").StringVar(&f.System.Host) + cmd.Flag("system-proto", "server proto").Default("").StringVar(&f.System.Proto) + cmd.Flag("system-link", "server link").Default("").StringVar(&f.System.Link) + cmd.Flag("system-version", "server version").Default("").StringVar(&f.System.Version) + + return f +} diff --git a/docker/Docker.windows.amd64.1809 b/docker/Docker.windows.amd64.1809 new file mode 100644 index 0000000..9517493 --- /dev/null +++ b/docker/Docker.windows.amd64.1809 @@ -0,0 +1,9 @@ +# escape=` +FROM mcr.microsoft.com/windows/nanoserver:1809 +USER ContainerAdministrator + +EXPOSE 3000 +ENV GODEBUG=netdns=go + +ADD release/windows/amd64/drone-runner-docker.exe C:/drone-runner-docker.exe +ENTRYPOINT [ "C:\\drone-runner-docker.exe" ] diff --git a/docker/Docker.windows.amd64.1903 b/docker/Docker.windows.amd64.1903 new file mode 100644 index 0000000..e04524b --- /dev/null +++ b/docker/Docker.windows.amd64.1903 @@ -0,0 +1,9 @@ +# escape=` +FROM mcr.microsoft.com/windows/nanoserver:1903 +USER ContainerAdministrator + +EXPOSE 3000 +ENV GODEBUG=netdns=go + +ADD release/windows/amd64/drone-runner-docker.exe C:/drone-runner-docker.exe +ENTRYPOINT [ "C:\\drone-runner-docker.exe" ] \ No newline at end of file diff --git a/docker/Dockerfile.linux.amd64 b/docker/Dockerfile.linux.amd64 new file mode 100644 index 0000000..f77f2c4 --- /dev/null +++ b/docker/Dockerfile.linux.amd64 @@ -0,0 +1,12 @@ +FROM alpine:3.6 as alpine +RUN apk add -U --no-cache ca-certificates + +FROM alpine:3.6 +EXPOSE 3000 + +ENV GODEBUG netdns=go + +COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +ADD release/linux/amd64/drone-runner-docker /bin/ +ENTRYPOINT ["/bin/drone-runner-docker"] diff --git a/docker/Dockerfile.linux.arm b/docker/Dockerfile.linux.arm new file mode 100644 index 0000000..d2a3a37 --- /dev/null +++ b/docker/Dockerfile.linux.arm @@ -0,0 +1,12 @@ +FROM alpine:3.6 as alpine +RUN apk add -U --no-cache ca-certificates + +FROM alpine:3.6 +EXPOSE 3000 + +ENV GODEBUG netdns=go + +COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +ADD release/linux/arm/drone-runner-docker /bin/ +ENTRYPOINT ["/bin/drone-runner-docker"] diff --git a/docker/Dockerfile.linux.arm64 b/docker/Dockerfile.linux.arm64 new file mode 100644 index 0000000..9f0e2b5 --- /dev/null +++ b/docker/Dockerfile.linux.arm64 @@ -0,0 +1,12 @@ +FROM alpine:3.6 as alpine +RUN apk add -U --no-cache ca-certificates + +FROM alpine:3.6 +EXPOSE 3000 + +ENV GODEBUG netdns=go + +COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +ADD release/linux/arm64/drone-runner-docker /bin/ +ENTRYPOINT ["/bin/drone-runner-docker"] diff --git a/engine/compiler/clone.go b/engine/compiler/clone.go new file mode 100644 index 0000000..a268966 --- /dev/null +++ b/engine/compiler/clone.go @@ -0,0 +1,53 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "strconv" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/runner-go/manifest" +) + +// default name of the clone step. +const cloneStepName = "clone" + +// helper function returns the clone image based on the +// target operating system. +func cloneImage(platform manifest.Platform) string { + switch platform.OS { + case "windows": + return "drone/git:latest" + default: + return "drone/git:1" + } +} + +// helper function configures the clone depth parameter, +// specific to the clone plugin. +func cloneParams(src manifest.Clone) map[string]string { + dst := map[string]string{} + if depth := src.Depth; depth > 0 { + dst["PLUGIN_DEPTH"] = strconv.Itoa(depth) + } + if skipVerify := src.SkipVerify; skipVerify { + dst["GIT_SSL_NO_VERIFY"] = "true" + dst["PLUGIN_SKIP_VERIFY"] = "true" + } + return dst +} + +// helper function creates a default container configuration +// for the clone stage. The clone stage is automatically +// added to each pipeline. +func createClone(src *resource.Pipeline) *engine.Step { + return &engine.Step{ + Name: cloneStepName, + Image: cloneImage(src.Platform), + RunPolicy: engine.RunAlways, + Envs: cloneParams(src.Clone), + } +} diff --git a/engine/compiler/clone_test.go b/engine/compiler/clone_test.go new file mode 100644 index 0000000..be4f2f5 --- /dev/null +++ b/engine/compiler/clone_test.go @@ -0,0 +1,126 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "testing" + + "github.com/dchest/uniuri" + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/manifest" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestClone(t *testing.T) { + random = notRandom + defer func() { + random = uniuri.New + }() + + c := &Compiler{ + Repo: &drone.Repo{}, + Build: &drone.Build{}, + Stage: &drone.Stage{}, + System: &drone.System{}, + Netrc: &drone.Netrc{}, + Manifest: &manifest.Manifest{}, + Pipeline: &resource.Pipeline{}, + } + want := []*engine.Step{ + { + ID: "random", + Image: "drone/git:1", + Name: "clone", + RunPolicy: engine.RunAlways, + WorkingDir: "/drone/src", + }, + } + got := c.Compile(nil) + ignore := cmpopts.IgnoreFields(engine.Step{}, "Envs") + if diff := cmp.Diff(got.Steps, want, ignore); len(diff) != 0 { + t.Errorf(diff) + } +} + +func TestCloneDisable(t *testing.T) { + c := &Compiler{ + Repo: &drone.Repo{}, + Build: &drone.Build{}, + Stage: &drone.Stage{}, + System: &drone.System{}, + Netrc: &drone.Netrc{}, + Manifest: &manifest.Manifest{}, + Pipeline: &resource.Pipeline{Clone: manifest.Clone{Disable: true}}, + } + got := c.Compile(nil) + if len(got.Steps) != 0 { + t.Errorf("Expect no clone step added when disabled") + } +} + +func TestCloneCreate(t *testing.T) { + want := &engine.Step{ + Name: "clone", + Image: "drone/git:1", + RunPolicy: engine.RunAlways, + Envs: map[string]string{"PLUGIN_DEPTH": "50"}, + } + src := &resource.Pipeline{Clone: manifest.Clone{Depth: 50}} + got := createClone(src) + if diff := cmp.Diff(got, want); len(diff) != 0 { + t.Errorf(diff) + } +} + +func TestCloneImage(t *testing.T) { + tests := []struct { + in manifest.Platform + out string + }{ + { + in: manifest.Platform{}, + out: "drone/git:1", + }, + { + in: manifest.Platform{OS: "linux"}, + out: "drone/git:1", + }, + { + in: manifest.Platform{OS: "windows"}, + out: "drone/git:latest", + }, + } + for _, test := range tests { + got, want := cloneImage(test.in), test.out + if got != want { + t.Errorf("Want clone image %q, got %q", want, got) + } + } +} + +func TestCloneParams(t *testing.T) { + params := cloneParams(manifest.Clone{}) + if len(params) != 0 { + t.Errorf("Expect empty clone parameters") + } + params = cloneParams(manifest.Clone{Depth: 0}) + if len(params) != 0 { + t.Errorf("Expect zero depth ignored") + } + params = cloneParams(manifest.Clone{Depth: 50, SkipVerify: true}) + if params["PLUGIN_DEPTH"] != "50" { + t.Errorf("Expect clone depth 50") + } + if params["GIT_SSL_NO_VERIFY"] != "true" { + t.Errorf("Expect GIT_SSL_NO_VERIFY is true") + } + if params["PLUGIN_SKIP_VERIFY"] != "true" { + t.Errorf("Expect PLUGIN_SKIP_VERIFY is true") + } +} diff --git a/engine/compiler/compiler.go b/engine/compiler/compiler.go new file mode 100644 index 0000000..6274f5b --- /dev/null +++ b/engine/compiler/compiler.go @@ -0,0 +1,238 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "context" + "fmt" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/clone" + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/secret" + + "github.com/dchest/uniuri" +) + +// random generator function +var random = uniuri.New + +// Compiler compiles the Yaml configuration file to an +// intermediate representation optimized for simple execution. +type Compiler struct { + // Manifest provides the parsed manifest. + Manifest *manifest.Manifest + + // Pipeline provides the parsed pipeline. This pipeline is + // the compiler source and is converted to the intermediate + // representation by the Compile method. + Pipeline *resource.Pipeline + + // Build provides the compiler with stage information that + // is converted to environment variable format and passed to + // each pipeline step. It is also used to clone the commit. + Build *drone.Build + + // Stage provides the compiler with stage information that + // is converted to environment variable format and passed to + // each pipeline step. + Stage *drone.Stage + + // Repo provides the compiler with repo information. This + // repo information is converted to environment variable + // format and passed to each pipeline step. It is also used + // to clone the repository. + Repo *drone.Repo + + // System provides the compiler with system information that + // is converted to environment variable format and passed to + // each pipeline step. + System *drone.System + + // Environ provides a set of environment variables that + // should be added to each pipeline step by default. + Environ map[string]string + + // Netrc provides netrc parameters that can be used by the + // default clone step to authenticate to the remote + // repository. + Netrc *drone.Netrc + + // Secret returns a named secret value that can be injected + // into the pipeline step. + Secret secret.Provider +} + +// Compile compiles the configuration file. +func (c *Compiler) Compile(ctx context.Context) *engine.Spec { + os := c.Pipeline.Platform.OS + + // create the workspace paths + base, path, full := createWorkspace(c.Pipeline) + + // create the workspace mount + mount := &engine.VolumeMount{ + Name: "_workspace", + Path: base, + } + + // create the workspace volume + volume := &engine.VolumeEmptyDir{ + ID: random(), + Name: mount.Name, + } + + spec := &engine.Spec{ + Network: engine.Network{ + ID: random(), + }, + Platform: engine.Platform{ + OS: c.Pipeline.Platform.OS, + Arch: c.Pipeline.Platform.Arch, + Variant: c.Pipeline.Platform.Variant, + Version: c.Pipeline.Platform.Version, + }, + Volumes: []*engine.Volume{ + {EmptyDir: volume}, + }, + } + + // create the default environment variables. + envs := environ.Combine( + c.Environ, + c.Build.Params, + environ.Proxy(), + environ.System(c.System), + environ.Repo(c.Repo), + environ.Build(c.Build), + environ.Stage(c.Stage), + environ.Link(c.Repo, c.Build, c.System), + clone.Environ(clone.Config{ + SkipVerify: c.Pipeline.Clone.SkipVerify, + Trace: c.Pipeline.Clone.Trace, + User: clone.User{ + Name: c.Build.AuthorName, + Email: c.Build.AuthorEmail, + }, + }), + ) + + // create docker reference variables + envs["DRONE_DOCKER_VOLUME_ID"] = volume.ID + envs["DRONE_DOCKER_NETWORK_ID"] = spec.Network.ID + + // create the workspace variables + envs["DRONE_WORKSPACE"] = full + envs["DRONE_WORKSPACE_BASE"] = base + envs["DRONE_WORKSPACE_PATH"] = path + + // create the netrc environment variables + if c.Netrc != nil && c.Netrc.Machine != "" { + envs["DRONE_NETRC_MACHINE"] = c.Netrc.Machine + envs["DRONE_NETRC_USERNAME"] = c.Netrc.Login + envs["DRONE_NETRC_PASSWORD"] = c.Netrc.Password + envs["DRONE_NETRC_FILE"] = fmt.Sprintf( + "machine %s login %s password %s", + c.Netrc.Machine, + c.Netrc.Login, + c.Netrc.Password, + ) + } + + match := manifest.Match{ + Action: c.Build.Action, + Cron: c.Build.Cron, + Ref: c.Build.Ref, + Repo: c.Repo.Slug, + Instance: c.System.Host, + Target: c.Build.Deploy, + Event: c.Build.Event, + Branch: c.Build.Target, + } + + // create the clone step + if c.Pipeline.Clone.Disable == false { + step := createClone(c.Pipeline) + step.ID = random() + step.Envs = environ.Combine(envs, step.Envs) + step.WorkingDir = full + step.Volumes = append(step.Volumes, mount) + spec.Steps = append(spec.Steps, step) + } + + // create steps + for _, src := range c.Pipeline.Services { + dst := createStep(c.Pipeline, src) + dst.Detach = true + dst.Envs = environ.Combine(envs, dst.Envs) + dst.Volumes = append(dst.Volumes, mount) + setupScript(src, dst, os) + setupWorkdir(src, dst, full) + spec.Steps = append(spec.Steps, dst) + + // if the pipeline step has unmet conditions the step is + // automatically skipped. + if !src.When.Match(match) { + dst.RunPolicy = engine.RunNever + } + } + + // create steps + for _, src := range c.Pipeline.Steps { + dst := createStep(c.Pipeline, src) + dst.Envs = environ.Combine(envs, dst.Envs) + dst.Volumes = append(dst.Volumes, mount) + setupScript(src, dst, full) + setupWorkdir(src, dst, full) + spec.Steps = append(spec.Steps, dst) + + // if the pipeline step has unmet conditions the step is + // automatically skipped. + if !src.When.Match(match) { + dst.RunPolicy = engine.RunNever + } + } + + if isGraph(spec) == false { + configureSerial(spec) + } else if c.Pipeline.Clone.Disable == false { + configureCloneDeps(spec) + } else if c.Pipeline.Clone.Disable == true { + removeCloneDeps(spec) + } + + for _, step := range spec.Steps { + for _, s := range step.Secrets { + secret, ok := c.findSecret(ctx, s.Name) + if ok { + s.Data = []byte(secret) + } + } + } + + return spec +} + +// helper function attempts to find and return the named secret. +// from the secret provider. +func (c *Compiler) findSecret(ctx context.Context, name string) (s string, ok bool) { + if name == "" { + return + } + found, _ := c.Secret.Find(ctx, &secret.Request{ + Name: name, + Build: c.Build, + Repo: c.Repo, + Conf: c.Manifest, + }) + if found == nil { + return + } + return found.Data, true +} diff --git a/engine/compiler/compiler_test.go b/engine/compiler/compiler_test.go new file mode 100644 index 0000000..b2d88f9 --- /dev/null +++ b/engine/compiler/compiler_test.go @@ -0,0 +1,186 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +// +build !windows + +package compiler + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "testing" + + "github.com/dchest/uniuri" + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/secret" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var nocontext = context.Background() + +// dummy function that returns a non-random string for testing. +// it is used in place of the random function. +func notRandom() string { + return "random" +} + +// This test verifies the pipeline dependency graph. When no +// dependency graph is defined, a default dependency graph is +// automatically defined to run steps serially. +func TestCompile_Serial(t *testing.T) { + testCompile(t, "testdata/serial.yml", "testdata/serial.json") +} + +// This test verifies the pipeline dependency graph. It also +// verifies that pipeline steps with no dependencies depend on +// the initial clone step. +func TestCompile_Graph(t *testing.T) { + testCompile(t, "testdata/graph.yml", "testdata/graph.json") +} + +// This test verifies no clone step exists in the pipeline if +// cloning is disabled. +func TestCompile_CloneDisabled_Serial(t *testing.T) { + testCompile(t, "testdata/noclone_serial.yml", "testdata/noclone_serial.json") +} + +// This test verifies no clone step exists in the pipeline if +// cloning is disabled. It also verifies no pipeline steps +// depend on a clone step. +func TestCompile_CloneDisabled_Graph(t *testing.T) { + testCompile(t, "testdata/noclone_graph.yml", "testdata/noclone_graph.json") +} + +// This test verifies that steps are disabled if conditions +// defined in the when block are not satisfied. +func TestCompile_Match(t *testing.T) { + ir := testCompile(t, "testdata/match.yml", "testdata/match.json") + if ir.Steps[0].RunPolicy != engine.RunOnSuccess { + t.Errorf("Expect run on success") + } + if ir.Steps[1].RunPolicy != engine.RunNever { + t.Errorf("Expect run never") + } +} + +// This test verifies that steps configured to run on both +// success or failure are configured to always run. +func TestCompile_RunAlways(t *testing.T) { + ir := testCompile(t, "testdata/run_always.yml", "testdata/run_always.json") + if ir.Steps[0].RunPolicy != engine.RunAlways { + t.Errorf("Expect run always") + } +} + +// This test verifies that steps configured to run on failure +// are configured to run on failure. +func TestCompile_RunFaiure(t *testing.T) { + ir := testCompile(t, "testdata/run_failure.yml", "testdata/run_failure.json") + if ir.Steps[0].RunPolicy != engine.RunOnFailure { + t.Errorf("Expect run on failure") + } +} + +// This test verifies that secrets defined in the yaml are +// requested and stored in the intermediate representation +// at compile time. +func TestCompile_Secrets(t *testing.T) { + manifest, _ := manifest.ParseFile("testdata/secret.yml") + compiler := Compiler{} + compiler.Build = &drone.Build{} + compiler.Repo = &drone.Repo{} + compiler.Stage = &drone.Stage{} + compiler.System = &drone.System{} + compiler.Netrc = &drone.Netrc{} + compiler.Manifest = manifest + compiler.Pipeline = manifest.Resources[0].(*resource.Pipeline) + compiler.Secret = secret.StaticVars(map[string]string{ + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "password": "password", + "my_username": "octocat", + }) + ir := compiler.Compile(nocontext) + got := ir.Steps[0].Secrets + want := []*engine.Secret{ + { + Name: "my_password", + Env: "PASSWORD", + Data: nil, // secret not found, data nil + Mask: true, + }, + { + Name: "my_username", + Env: "USERNAME", + Data: []byte("octocat"), // secret found + Mask: true, + }, + } + if diff := cmp.Diff(got, want); len(diff) != 0 { + // TODO(bradrydzewski) ordering is not guaranteed. this + // unit tests needs to be adjusted accordingly. + t.Skipf(diff) + } +} + +// helper function parses and compiles the source file and then +// compares to a golden json file. +func testCompile(t *testing.T, source, golden string) *engine.Spec { + // replace the default random function with one that + // is deterministic, for testing purposes. + random = notRandom + + // restore the default random function and the previously + // specified temporary directory + defer func() { + random = uniuri.New + }() + + manifest, err := manifest.ParseFile(source) + if err != nil { + t.Error(err) + return nil + } + + compiler := Compiler{} + compiler.Build = &drone.Build{Target: "master"} + compiler.Repo = &drone.Repo{} + compiler.Stage = &drone.Stage{} + compiler.System = &drone.System{} + compiler.Netrc = &drone.Netrc{Machine: "github.com", Login: "octocat", Password: "correct-horse-battery-staple"} + compiler.Manifest = manifest + compiler.Pipeline = manifest.Resources[0].(*resource.Pipeline) + got := compiler.Compile(nocontext) + + raw, err := ioutil.ReadFile(golden) + if err != nil { + t.Error(err) + } + + want := new(engine.Spec) + err = json.Unmarshal(raw, want) + if err != nil { + t.Error(err) + } + + ignore := cmpopts.IgnoreFields(engine.Step{}, "Envs", "Secrets") + unexported := cmpopts.IgnoreUnexported(engine.Spec{}) + if diff := cmp.Diff(got, want, ignore, unexported); len(diff) != 0 { + t.Errorf(diff) + } + + return got +} + +func dump(v interface{}) { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(v) +} diff --git a/engine/compiler/encoder/encoder.go b/engine/compiler/encoder/encoder.go new file mode 100644 index 0000000..8b261c2 --- /dev/null +++ b/engine/compiler/encoder/encoder.go @@ -0,0 +1,56 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package encoder + +import ( + "encoding/base64" + "strconv" + "strings" + + "github.com/buildkite/yaml" + json "github.com/ghodss/yaml" +) + +// Encode encodes an interface value as a string. This function +// assumes all types were unmarshaled by the yaml.v2 library. +// The yaml.v2 package only supports a subset of primative types. +func Encode(v interface{}) string { + switch v := v.(type) { + case string: + return v + case bool: + return strconv.FormatBool(v) + case int: + return strconv.Itoa(v) + case float64: + return strconv.FormatFloat(v, 'g', -1, 64) + case []byte: + return base64.StdEncoding.EncodeToString(v) + case []interface{}: + return encodeSlice(v) + default: + return encodeMap(v) + } +} + +// helper function encodes a parameter in map format. +func encodeMap(v interface{}) string { + yml, _ := yaml.Marshal(v) + out, _ := json.YAMLToJSON(yml) + return string(out) +} + +// helper function encodes a parameter in slice format. +func encodeSlice(v interface{}) string { + out, _ := yaml.Marshal(v) + + in := []string{} + err := yaml.Unmarshal(out, &in) + if err == nil { + return strings.Join(in, ",") + } + out, _ = json.YAMLToJSON(out) + return string(out) +} diff --git a/engine/compiler/encoder/encoder_test.go b/engine/compiler/encoder/encoder_test.go new file mode 100644 index 0000000..bfc7b20 --- /dev/null +++ b/engine/compiler/encoder/encoder_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package encoder + +import "testing" + +func TestEncode(t *testing.T) { + testdatum := []struct { + data interface{} + text string + }{ + { + data: "foo", + text: "foo", + }, + { + data: true, + text: "true", + }, + { + data: 42, + text: "42", + }, + { + data: float64(42.424242), + text: "42.424242", + }, + { + data: []interface{}{"foo", "bar", "baz"}, + text: "foo,bar,baz", + }, + { + data: []interface{}{1, 1, 2, 3, 5, 8}, + text: "1,1,2,3,5,8", + }, + { + data: []byte("foo"), + text: "Zm9v", + }, + { + data: []interface{}{ + struct { + Name string `json:"name"` + }{ + Name: "john", + }, + }, + text: `[{"name":"john"}]`, + }, + { + data: map[interface{}]interface{}{"foo": "bar"}, + text: `{"foo":"bar"}`, + }, + } + + for _, testdata := range testdatum { + if got, want := Encode(testdata.data), testdata.text; got != want { + t.Errorf("Want interface{} encoded to %q, got %q", want, got) + } + } +} diff --git a/engine/compiler/image/image.go b/engine/compiler/image/image.go new file mode 100644 index 0000000..8545e0f --- /dev/null +++ b/engine/compiler/image/image.go @@ -0,0 +1,71 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package image + +import "github.com/docker/distribution/reference" + +// Trim returns the short image name without tag. +func Trim(name string) string { + ref, err := reference.ParseAnyReference(name) + if err != nil { + return name + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return name + } + named = reference.TrimNamed(named) + return reference.FamiliarName(named) +} + +// Expand returns the fully qualified image name. +func Expand(name string) string { + ref, err := reference.ParseAnyReference(name) + if err != nil { + return name + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return name + } + named = reference.TagNameOnly(named) + return named.String() +} + +// Match returns true if the image name matches +// an image in the list. Note the image tag is not used +// in the matching logic. +func Match(from string, to ...string) bool { + from = Trim(from) + for _, match := range to { + if from == Trim(match) { + return true + } + } + return false +} + +// MatchTag returns true if the image name matches +// an image in the list, including the tag. +func MatchTag(a, b string) bool { + return Expand(a) == Expand(b) +} + +// MatchHostname returns true if the image hostname +// matches the specified hostname. +func MatchHostname(image, hostname string) bool { + ref, err := reference.ParseAnyReference(image) + if err != nil { + return false + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return false + } + if hostname == "index.docker.io" { + hostname = "docker.io" + } + return reference.Domain(named) == hostname +} diff --git a/engine/compiler/image/image_test.go b/engine/compiler/image/image_test.go new file mode 100644 index 0000000..eb252f4 --- /dev/null +++ b/engine/compiler/image/image_test.go @@ -0,0 +1,299 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package image + +import "testing" + +func Test_trimImage(t *testing.T) { + testdata := []struct { + from string + want string + }{ + { + from: "golang", + want: "golang", + }, + { + from: "golang:latest", + want: "golang", + }, + { + from: "golang:1.0.0", + want: "golang", + }, + { + from: "library/golang", + want: "golang", + }, + { + from: "library/golang:latest", + want: "golang", + }, + { + from: "library/golang:1.0.0", + want: "golang", + }, + { + from: "index.docker.io/library/golang:1.0.0", + want: "golang", + }, + { + from: "docker.io/library/golang:1.0.0", + want: "golang", + }, + { + from: "gcr.io/library/golang:1.0.0", + want: "gcr.io/library/golang", + }, + // error cases, return input unmodified + { + from: "foo/bar?baz:boo", + want: "foo/bar?baz:boo", + }, + } + for _, test := range testdata { + got, want := Trim(test.from), test.want + if got != want { + t.Errorf("Want image %q trimmed to %q, got %q", test.from, want, got) + } + } +} + +func Test_expandImage(t *testing.T) { + testdata := []struct { + from string + want string + }{ + { + from: "golang", + want: "docker.io/library/golang:latest", + }, + { + from: "golang:latest", + want: "docker.io/library/golang:latest", + }, + { + from: "golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "library/golang", + want: "docker.io/library/golang:latest", + }, + { + from: "library/golang:latest", + want: "docker.io/library/golang:latest", + }, + { + from: "library/golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "index.docker.io/library/golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "gcr.io/golang", + want: "gcr.io/golang:latest", + }, + { + from: "gcr.io/golang:1.0.0", + want: "gcr.io/golang:1.0.0", + }, + // error cases, return input unmodified + { + from: "foo/bar?baz:boo", + want: "foo/bar?baz:boo", + }, + } + for _, test := range testdata { + got, want := Expand(test.from), test.want + if got != want { + t.Errorf("Want image %q expanded to %q, got %q", test.from, want, got) + } + } +} + +func Test_matchImage(t *testing.T) { + testdata := []struct { + from, to string + want bool + }{ + { + from: "golang", + to: "golang", + want: true, + }, + { + from: "golang:latest", + to: "golang", + want: true, + }, + { + from: "library/golang:latest", + to: "golang", + want: true, + }, + { + from: "index.docker.io/library/golang:1.0.0", + to: "golang", + want: true, + }, + { + from: "golang", + to: "golang:latest", + want: true, + }, + { + from: "library/golang:latest", + to: "library/golang", + want: true, + }, + { + from: "gcr.io/golang", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang:1.0.0", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang:latest", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang", + to: "gcr.io/golang:latest", + want: true, + }, + { + from: "golang", + to: "library/golang", + want: true, + }, + { + from: "golang", + to: "gcr.io/project/golang", + want: false, + }, + { + from: "golang", + to: "gcr.io/library/golang", + want: false, + }, + { + from: "golang", + to: "gcr.io/golang", + want: false, + }, + } + for _, test := range testdata { + got, want := Match(test.from, test.to), test.want + if got != want { + t.Errorf("Want image %q matching %q is %v", test.from, test.to, want) + } + } +} + +func Test_matchHostname(t *testing.T) { + testdata := []struct { + image, hostname string + want bool + }{ + { + image: "golang", + hostname: "docker.io", + want: true, + }, + { + image: "golang:latest", + hostname: "docker.io", + want: true, + }, + { + image: "golang:latest", + hostname: "index.docker.io", + want: true, + }, + { + image: "library/golang:latest", + hostname: "docker.io", + want: true, + }, + { + image: "docker.io/library/golang:1.0.0", + hostname: "docker.io", + want: true, + }, + { + image: "gcr.io/golang", + hostname: "docker.io", + want: false, + }, + { + image: "gcr.io/golang:1.0.0", + hostname: "gcr.io", + want: true, + }, + { + image: "1.2.3.4:8000/golang:1.0.0", + hostname: "1.2.3.4:8000", + want: true, + }, + { + image: "*&^%", + hostname: "1.2.3.4:8000", + want: false, + }, + } + for _, test := range testdata { + got, want := MatchHostname(test.image, test.hostname), test.want + if got != want { + t.Errorf("Want image %q matching hostname %q is %v", test.image, test.hostname, want) + } + } +} + +func Test_matchTag(t *testing.T) { + testdata := []struct { + a, b string + want bool + }{ + { + a: "golang:1.0", + b: "golang:1.0", + want: true, + }, + { + a: "golang", + b: "golang:latest", + want: true, + }, + { + a: "docker.io/library/golang", + b: "golang:latest", + want: true, + }, + { + a: "golang", + b: "golang:1.0", + want: false, + }, + { + a: "golang:1.0", + b: "golang:2.0", + want: false, + }, + } + for _, test := range testdata { + got, want := MatchTag(test.a, test.b), test.want + if got != want { + t.Errorf("Want image %q matching image tag %q is %v", test.a, test.b, want) + } + } +} diff --git a/engine/compiler/os.go b/engine/compiler/os.go new file mode 100644 index 0000000..5522c39 --- /dev/null +++ b/engine/compiler/os.go @@ -0,0 +1,85 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "fmt" + "strings" + + "github.com/drone/runner-go/shell/bash" + "github.com/drone/runner-go/shell/powershell" +) + +// helper function returns the base temporary directory based +// on the target platform. +func tempdir(os string) string { + dir := fmt.Sprintf("drone-%s", random()) + switch os { + case "windows": + return join(os, "C:\\Windows\\Temp", dir) + default: + return join(os, "/tmp", dir) + } +} + +// helper function joins the file paths. +func join(os string, paths ...string) string { + switch os { + case "windows": + return strings.Join(paths, "\\") + default: + return strings.Join(paths, "/") + } +} + +// helper function returns the shell extension based on the +// target platform. +func getExt(os, file string) (s string) { + switch os { + case "windows": + return file + ".ps1" + default: + return file + } +} + +// +// TODO(bradrydzewski) can we remove the below functions? +// + +// helper function returns the shell command and arguments +// based on the target platform to invoke the script +func getCommand(os, script string) (string, []string) { + cmd, args := bash.Command() + switch os { + case "windows": + cmd, args = powershell.Command() + } + return cmd, append(args, script) +} + +// helper function returns the netrc file name based on the +// target platform. +func getNetrc(os string) string { + switch os { + case "windows": + return "_netrc" + default: + return ".netrc" + } +} + +// helper function generates and returns a shell script to +// execute the provided shell commands. The shell scripting +// language (bash vs pwoershell) is determined by the operating +// system. +func genScript(os string, commands []string) string { + switch os { + case "windows": + return powershell.Script(commands) + default: + return bash.Script(commands) + } +} diff --git a/engine/compiler/os_test.go b/engine/compiler/os_test.go new file mode 100644 index 0000000..9eb9bff --- /dev/null +++ b/engine/compiler/os_test.go @@ -0,0 +1,128 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "reflect" + "testing" + + "github.com/drone/runner-go/shell/bash" + "github.com/drone/runner-go/shell/powershell" + + "github.com/dchest/uniuri" +) + +func Test_tempdir(t *testing.T) { + // replace the default random function with one that + // is deterministic, for testing purposes. + random = notRandom + + // restore the default random function and the previously + // specified temporary directory + defer func() { + random = uniuri.New + }() + + tests := []struct { + os string + path string + }{ + {os: "windows", path: "C:\\Windows\\Temp\\drone-random"}, + {os: "linux", path: "/tmp/drone-random"}, + {os: "openbsd", path: "/tmp/drone-random"}, + {os: "netbsd", path: "/tmp/drone-random"}, + {os: "freebsd", path: "/tmp/drone-random"}, + } + + for _, test := range tests { + if got, want := tempdir(test.os), test.path; got != want { + t.Errorf("Want tempdir %s, got %s", want, got) + } + } +} + +func Test_join(t *testing.T) { + tests := []struct { + os string + a []string + b string + }{ + {os: "windows", a: []string{"C:", "Windows", "Temp"}, b: "C:\\Windows\\Temp"}, + {os: "linux", a: []string{"/tmp", "foo", "bar"}, b: "/tmp/foo/bar"}, + } + for _, test := range tests { + if got, want := join(test.os, test.a...), test.b; got != want { + t.Errorf("Want %s, got %s", want, got) + } + } +} + +func Test_getExt(t *testing.T) { + tests := []struct { + os string + a string + b string + }{ + {os: "windows", a: "clone", b: "clone.ps1"}, + {os: "linux", a: "clone", b: "clone"}, + } + for _, test := range tests { + if got, want := getExt(test.os, test.a), test.b; got != want { + t.Errorf("Want %s, got %s", want, got) + } + } +} + +func Test_getCommand(t *testing.T) { + cmd, args := getCommand("linux", "clone.sh") + if got, want := cmd, "/bin/sh"; got != want { + t.Errorf("Want command %s, got %s", want, got) + } + if !reflect.DeepEqual(args, []string{"-e", "clone.sh"}) { + t.Errorf("Unexpected args %v", args) + } + + cmd, args = getCommand("windows", "clone.ps1") + if got, want := cmd, "powershell"; got != want { + t.Errorf("Want command %s, got %s", want, got) + } + if !reflect.DeepEqual(args, []string{"-noprofile", "-noninteractive", "-command", "clone.ps1"}) { + t.Errorf("Unexpected args %v", args) + } +} + +func Test_getNetrc(t *testing.T) { + tests := []struct { + os string + name string + }{ + {os: "windows", name: "_netrc"}, + {os: "linux", name: ".netrc"}, + {os: "openbsd", name: ".netrc"}, + {os: "netbsd", name: ".netrc"}, + {os: "freebsd", name: ".netrc"}, + } + for _, test := range tests { + if got, want := getNetrc(test.os), test.name; got != want { + t.Errorf("Want %s on %s, got %s", want, test.os, got) + } + } +} + +func Test_getScript(t *testing.T) { + commands := []string{"go build"} + + a := genScript("windows", commands) + b := powershell.Script(commands) + if !reflect.DeepEqual(a, b) { + t.Errorf("Generated windows linux script") + } + + a = genScript("linux", commands) + b = bash.Script(commands) + if !reflect.DeepEqual(a, b) { + t.Errorf("Generated invalid linux script") + } +} diff --git a/engine/compiler/script.go b/engine/compiler/script.go new file mode 100644 index 0000000..f9748d3 --- /dev/null +++ b/engine/compiler/script.go @@ -0,0 +1,35 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/compiler/shell" + "github.com/drone-runners/drone-runner-docker/engine/compiler/shell/powershell" + "github.com/drone-runners/drone-runner-docker/engine/resource" +) + +func setupScript(src *resource.Step, dst *engine.Step, os string) { + if len(src.Commands) > 0 { + switch os { + case "windows": + setupScriptWindows(src, dst) + default: + setupScriptPosix(src, dst) + } + } +} + +func setupScriptWindows(src *resource.Step, dst *engine.Step) { + dst.Entrypoint = []string{"powershell", "-noprofile", "-noninteractive", "-command"} + dst.Command = []string{"echo $DRONE_SCRIPT | iex"} + dst.Envs["DRONE_SCRIPT"] = powershell.Script(src.Commands) +} + +func setupScriptPosix(src *resource.Step, dst *engine.Step) { + dst.Entrypoint = []string{"/bin/sh", "-c"} + dst.Command = []string{"echo $DRONE_SCRIPT | /bin/sh -e"} + dst.Envs["DRONE_SCRIPT"] = shell.Script(src.Commands) +} diff --git a/engine/compiler/script_test.go b/engine/compiler/script_test.go new file mode 100644 index 0000000..4bc3980 --- /dev/null +++ b/engine/compiler/script_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler diff --git a/engine/compiler/shell/powershell/powershell.go b/engine/compiler/shell/powershell/powershell.go new file mode 100644 index 0000000..0de71d1 --- /dev/null +++ b/engine/compiler/shell/powershell/powershell.go @@ -0,0 +1,59 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +// Package powershell provides functions for converting shell +// commands to powershell scripts. +package powershell + +import ( + "bytes" + "fmt" + "strings" +) + +// Script converts a slice of individual shell commands to +// a powershell script. +func Script(commands []string) string { + buf := new(bytes.Buffer) + fmt.Fprintln(buf) + fmt.Fprintf(buf, optionScript) + fmt.Fprintln(buf) + for _, command := range commands { + escaped := fmt.Sprintf("%q", "+ "+command) + escaped = strings.Replace(escaped, "$", "`$", -1) + buf.WriteString(fmt.Sprintf( + traceScript, + escaped, + command, + )) + } + return buf.String() +} + +// optionScript is a helper script this is added to the build +// to set shell options, in this case, to exit on error. +const optionScript = ` +if ($Env:DRONE_NETRC_MACHINE) { +@" +machine $Env:DRONE_NETRC_MACHINE +login $Env:DRONE_NETRC_USERNAME +password $Env:DRONE_NETRC_PASSWORD +"@ > (Join-Path $Env:USERPROFILE '_netrc'); +} +[Environment]::SetEnvironmentVariable("DRONE_NETRC_USERNAME", $null); +[Environment]::SetEnvironmentVariable("DRONE_NETRC_PASSWORD", $null); +[Environment]::SetEnvironmentVariable("DRONE_NETRC_USERNAME", $null); +[Environment]::SetEnvironmentVariable("DRONE_NETRC_PASSWORD", $null); +[Environment]::SetEnvironmentVariable("DRONE_SCRIPT", $null); + +$erroractionpreference = "stop" +` + +// traceScript is a helper script that is added to +// the build script to trace a command. +const traceScript = ` +echo %s +%s +if ($LastExitCode -ne 0) { exit $LastExitCode } +` diff --git a/engine/compiler/shell/powershell/powershell_test.go b/engine/compiler/shell/powershell/powershell_test.go new file mode 100644 index 0000000..573b3cc --- /dev/null +++ b/engine/compiler/shell/powershell/powershell_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package powershell diff --git a/engine/compiler/shell/shell.go b/engine/compiler/shell/shell.go new file mode 100644 index 0000000..e779221 --- /dev/null +++ b/engine/compiler/shell/shell.go @@ -0,0 +1,56 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +// Package shell provides functions for converting shell commands +// to posix shell scripts. +package shell + +import ( + "bytes" + "fmt" + "strings" +) + +// Script converts a slice of individual shell commands to +// a posix-compliant shell script. +func Script(commands []string) string { + buf := new(bytes.Buffer) + fmt.Fprintln(buf) + fmt.Fprintf(buf, optionScript) + fmt.Fprintln(buf) + for _, command := range commands { + escaped := fmt.Sprintf("%q", command) + escaped = strings.Replace(escaped, "$", `\$`, -1) + buf.WriteString(fmt.Sprintf( + traceScript, + escaped, + command, + )) + } + return buf.String() +} + +// optionScript is a helper script this is added to the build +// to set shell options, in this case, to exit on error. +const optionScript = ` +if [[ ! -z "${DRONE_NETRC_FILE}" ]]; then + echo $DRONE_NETRC_FILE > $HOME/.netrc +EOF +fi + +unset DRONE_SCRIPT +unset DRONE_NETRC_MACHINE +unset DRONE_NETRC_USERNAME +unset DRONE_NETRC_PASSWORD +unset DRONE_NETRC_FILE + +set -e +` + +// traceScript is a helper script that is added to +// the build script to trace a command. +const traceScript = ` +echo + %s +%s +` diff --git a/engine/compiler/shell/shell_test.go b/engine/compiler/shell/shell_test.go new file mode 100644 index 0000000..8ad768e --- /dev/null +++ b/engine/compiler/shell/shell_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package shell diff --git a/engine/compiler/step.go b/engine/compiler/step.go new file mode 100644 index 0000000..9ed1d19 --- /dev/null +++ b/engine/compiler/step.go @@ -0,0 +1,108 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "strings" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/compiler/encoder" + "github.com/drone-runners/drone-runner-docker/engine/compiler/image" + "github.com/drone-runners/drone-runner-docker/engine/resource" +) + +func createStep(spec *resource.Pipeline, src *resource.Step) *engine.Step { + dst := &engine.Step{ + ID: random(), + Name: src.Name, + Image: image.Expand(src.Image), + Command: src.Command, + Entrypoint: src.Entrypoint, + Detach: src.Detach, + DependsOn: src.DependsOn, + DNS: src.DNS, + DNSSearch: src.DNSSearch, + Envs: convertStaticEnv(src.Environment), + ExtraHosts: src.ExtraHosts, + IgnoreErr: strings.EqualFold(src.Failure, "ignore"), + IgnoreStderr: false, + IgnoreStdout: false, + Network: src.Network, + Privileged: src.Privileged, + Pull: convertPullPolicy(src.Pull), + User: src.User, + Secrets: convertSecretEnv(src.Environment), + WorkingDir: src.WorkingDir, + + // + // + // + + Networks: nil, // set in compiler.go + Files: nil, // set below + Volumes: nil, // set below + // Devices: nil, // TODO + // Resources: toResources(src), // TODO + } + + // appends the volumes to the container def. + for _, vol := range src.Volumes { + dst.Volumes = append(dst.Volumes, &engine.VolumeMount{ + Name: vol.Name, + Path: vol.MountPath, + }) + } + + // appends the settings variables to the + // container definition. + for key, value := range src.Settings { + // fix https://github.com/drone/drone-yaml/issues/13 + if value == nil { + continue + } + // all settings are passed to the plugin env + // variables, prefixed with PLUGIN_ + key = "PLUGIN_" + strings.ToUpper(key) + + // if the setting parameter is sources from the + // secret we create a secret enviornment variable. + if value.Secret != "" { + dst.Secrets = append(dst.Secrets, &engine.Secret{ + Name: value.Secret, + Mask: true, + Env: key, + }) + } else { + // else if the setting parameter is opaque + // we inject as a string-encoded environment + // variable. + dst.Envs[key] = encoder.Encode(value.Value) + } + } + + // // if the step specifies shell commands we generate a + // // script. The script is copied to the container at + // // runtime (or mounted as a config map) and then executed + // // as the entrypoint. + // if len(src.Commands) > 0 { + // switch spec.Platform.OS { + // case "windows": + // setupScriptWin(spec, dst, src) + // default: + // setupScript(spec, dst, src) + // } + // } + + // set the pipeline step run policy. steps run on + // success by default, but may be optionally configured + // to run on failure. + if isRunAlways(src) { + dst.RunPolicy = engine.RunAlways + } else if isRunOnFailure(src) { + dst.RunPolicy = engine.RunOnFailure + } + + return dst +} diff --git a/engine/compiler/testdata/graph.json b/engine/compiler/testdata/graph.json new file mode 100644 index 0000000..33e15cd --- /dev/null +++ b/engine/compiler/testdata/graph.json @@ -0,0 +1,104 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/clone" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/clone", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnaXQgaW5pdCIKZ2l0IGluaXQKCmVjaG8gKyAiZ2l0IHJlbW90ZSBhZGQgb3JpZ2luICIKZ2l0IHJlbW90ZSBhZGQgb3JpZ2luIAoKZWNobyArICJnaXQgZmV0Y2ggIG9yaWdpbiArcmVmcy9oZWFkcy9tYXN0ZXI6IgpnaXQgZmV0Y2ggIG9yaWdpbiArcmVmcy9oZWFkcy9tYXN0ZXI6CgplY2hvICsgImdpdCBjaGVja291dCAgLWIgbWFzdGVyIgpnaXQgY2hlY2tvdXQgIC1iIG1hc3Rlcgo=" + } + ], + "secrets": [], + "name": "clone", + "run_policy": 2, + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "depends_on": [ + "clone" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "secrets": [], + "name": "build", + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/test" + ], + "command": "/bin/sh", + "depends_on": [ + "build" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/test", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyB0ZXN0IgpnbyB0ZXN0Cg==" + } + ], + "secrets": [], + "name": "test", + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} \ No newline at end of file diff --git a/engine/compiler/testdata/graph.yml b/engine/compiler/testdata/graph.yml new file mode 100644 index 0000000..dcf7775 --- /dev/null +++ b/engine/compiler/testdata/graph.yml @@ -0,0 +1,20 @@ +kind: pipeline +type: docker +name: default + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + +- name: test + commands: + - go test + depends_on: [ build ] diff --git a/engine/compiler/testdata/match.json b/engine/compiler/testdata/match.json new file mode 100644 index 0000000..a6e7c24 --- /dev/null +++ b/engine/compiler/testdata/match.json @@ -0,0 +1,82 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "name": "build", + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/test" + ], + "command": "/bin/sh", + "depends_on": [ + "build" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/test", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyB0ZXN0IgpnbyB0ZXN0Cg==" + } + ], + "name": "test", + "run_policy": 3, + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} \ No newline at end of file diff --git a/engine/compiler/testdata/match.yml b/engine/compiler/testdata/match.yml new file mode 100644 index 0000000..ec3a34a --- /dev/null +++ b/engine/compiler/testdata/match.yml @@ -0,0 +1,26 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + when: + branch: [ master ] + +- name: test + commands: + - go test + when: + branch: [ develop ] diff --git a/engine/compiler/testdata/noclone_graph.json b/engine/compiler/testdata/noclone_graph.json new file mode 100644 index 0000000..6b4019d --- /dev/null +++ b/engine/compiler/testdata/noclone_graph.json @@ -0,0 +1,83 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "name": "build", + "secrets": [], + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/test" + ], + "command": "/bin/sh", + "depends_on": [ + "build" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/test", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyB0ZXN0IgpnbyB0ZXN0Cg==" + } + ], + "name": "test", + "secrets": [], + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} \ No newline at end of file diff --git a/engine/compiler/testdata/noclone_graph.yml b/engine/compiler/testdata/noclone_graph.yml new file mode 100644 index 0000000..f3c5072 --- /dev/null +++ b/engine/compiler/testdata/noclone_graph.yml @@ -0,0 +1,23 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + +- name: test + commands: + - go test + depends_on: [ build ] \ No newline at end of file diff --git a/engine/compiler/testdata/noclone_serial.json b/engine/compiler/testdata/noclone_serial.json new file mode 100644 index 0000000..999cf11 --- /dev/null +++ b/engine/compiler/testdata/noclone_serial.json @@ -0,0 +1,62 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQKCmVjaG8gKyAiZ28gdGVzdCIKZ28gdGVzdAo=" + } + ], + "name": "build", + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} \ No newline at end of file diff --git a/engine/compiler/testdata/noclone_serial.yml b/engine/compiler/testdata/noclone_serial.yml new file mode 100644 index 0000000..48535ab --- /dev/null +++ b/engine/compiler/testdata/noclone_serial.yml @@ -0,0 +1,19 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + - go test diff --git a/engine/compiler/testdata/run_always.json b/engine/compiler/testdata/run_always.json new file mode 100644 index 0000000..94cebc4 --- /dev/null +++ b/engine/compiler/testdata/run_always.json @@ -0,0 +1,63 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "name": "build", + "run_policy": 2, + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} diff --git a/engine/compiler/testdata/run_always.yml b/engine/compiler/testdata/run_always.yml new file mode 100644 index 0000000..c46b3da --- /dev/null +++ b/engine/compiler/testdata/run_always.yml @@ -0,0 +1,20 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + when: + status: [ success, failure ] diff --git a/engine/compiler/testdata/run_failure.json b/engine/compiler/testdata/run_failure.json new file mode 100644 index 0000000..4422680 --- /dev/null +++ b/engine/compiler/testdata/run_failure.json @@ -0,0 +1,63 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "name": "build", + "run_policy": 1, + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} diff --git a/engine/compiler/testdata/run_failure.yml b/engine/compiler/testdata/run_failure.yml new file mode 100644 index 0000000..c99198d --- /dev/null +++ b/engine/compiler/testdata/run_failure.yml @@ -0,0 +1,20 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + when: + status: [ failure ] diff --git a/engine/compiler/testdata/secret.yml b/engine/compiler/testdata/secret.yml new file mode 100644 index 0000000..86b8ae2 --- /dev/null +++ b/engine/compiler/testdata/secret.yml @@ -0,0 +1,25 @@ +kind: pipeline +type: docker +name: default + +clone: + disable: true + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: + from_secret: token + +steps: +- name: build + environment: + PASSWORD: + from_secret: my_password + USERNAME: + from_secret: my_username + commands: + - go build + - go test diff --git a/engine/compiler/testdata/serial.json b/engine/compiler/testdata/serial.json new file mode 100644 index 0000000..33e15cd --- /dev/null +++ b/engine/compiler/testdata/serial.json @@ -0,0 +1,104 @@ +{ + "platform": {}, + "token": "3DA541559918A808C2402BBA5012F6C60B27661C", + "server": { + "name": "drone-temp-random", + "image": "docker-18-04", + "region": "nyc1", + "size": "s-1vcpu-1gb", + "user": "root" + }, + "root": "/tmp/drone-random", + "files": [ + { + "path": "/tmp/drone-random/home", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/drone/src", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/opt", + "mode": 448, + "is_dir": true + }, + { + "path": "/tmp/drone-random/home/drone/.netrc", + "mode": 384, + "data": "bWFjaGluZSBnaXRodWIuY29tIGxvZ2luIG9jdG9jYXQgcGFzc3dvcmQgY29ycmVjdC1ob3JzZS1iYXR0ZXJ5LXN0YXBsZQ==" + } + ], + "steps": [ + { + "args": [ + "-e", + "/tmp/drone-random/opt/clone" + ], + "command": "/bin/sh", + "files": [ + { + "path": "/tmp/drone-random/opt/clone", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnaXQgaW5pdCIKZ2l0IGluaXQKCmVjaG8gKyAiZ2l0IHJlbW90ZSBhZGQgb3JpZ2luICIKZ2l0IHJlbW90ZSBhZGQgb3JpZ2luIAoKZWNobyArICJnaXQgZmV0Y2ggIG9yaWdpbiArcmVmcy9oZWFkcy9tYXN0ZXI6IgpnaXQgZmV0Y2ggIG9yaWdpbiArcmVmcy9oZWFkcy9tYXN0ZXI6CgplY2hvICsgImdpdCBjaGVja291dCAgLWIgbWFzdGVyIgpnaXQgY2hlY2tvdXQgIC1iIG1hc3Rlcgo=" + } + ], + "secrets": [], + "name": "clone", + "run_policy": 2, + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/build" + ], + "command": "/bin/sh", + "depends_on": [ + "clone" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/build", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyBidWlsZCIKZ28gYnVpbGQK" + } + ], + "secrets": [], + "name": "build", + "working_dir": "/tmp/drone-random/drone/src" + }, + { + "args": [ + "-e", + "/tmp/drone-random/opt/test" + ], + "command": "/bin/sh", + "depends_on": [ + "build" + ], + "files": [ + { + "path": "/tmp/drone-random/opt/test", + "mode": 448, + "data": "CnNldCAtZQoKZWNobyArICJnbyB0ZXN0IgpnbyB0ZXN0Cg==" + } + ], + "secrets": [], + "name": "test", + "working_dir": "/tmp/drone-random/drone/src" + } + ] +} \ No newline at end of file diff --git a/engine/compiler/testdata/serial.yml b/engine/compiler/testdata/serial.yml new file mode 100644 index 0000000..9bf3010 --- /dev/null +++ b/engine/compiler/testdata/serial.yml @@ -0,0 +1,19 @@ +kind: pipeline +type: docker +name: default + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +token: 3DA541559918A808C2402BBA5012F6C60B27661C + +steps: +- name: build + commands: + - go build + +- name: test + commands: + - go test diff --git a/engine/compiler/util.go b/engine/compiler/util.go new file mode 100644 index 0000000..c8304c3 --- /dev/null +++ b/engine/compiler/util.go @@ -0,0 +1,132 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "strings" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/manifest" +) + +// helper function returns true if the step is configured to +// always run regardless of status. +func isRunAlways(step *resource.Step) bool { + if len(step.When.Status.Include) == 0 && + len(step.When.Status.Exclude) == 0 { + return false + } + return step.When.Status.Match(drone.StatusFailing) && + step.When.Status.Match(drone.StatusPassing) +} + +// helper function returns true if the step is configured to +// only run on failure. +func isRunOnFailure(step *resource.Step) bool { + if len(step.When.Status.Include) == 0 && + len(step.When.Status.Exclude) == 0 { + return false + } + return step.When.Status.Match(drone.StatusFailing) +} + +// helper function returns true if the pipeline specification +// manually defines an execution graph. +func isGraph(spec *engine.Spec) bool { + for _, step := range spec.Steps { + if len(step.DependsOn) > 0 { + return true + } + } + return false +} + +// helper function creates the dependency graph for serial +// pipeline execution. +func configureSerial(spec *engine.Spec) { + var prev *engine.Step + for _, step := range spec.Steps { + if prev != nil { + step.DependsOn = []string{prev.Name} + } + prev = step + } +} + +// helper function converts the environment variables to a map, +// returning only inline environment variables not derived from +// a secret. +func convertStaticEnv(src map[string]*manifest.Variable) map[string]string { + dst := map[string]string{} + for k, v := range src { + if strings.TrimSpace(v.Secret) == "" { + dst[k] = v.Value + } + } + return dst +} + +// helper function converts the environment variables to a map, +// returning only inline environment variables not derived from +// a secret. +func convertSecretEnv(src map[string]*manifest.Variable) []*engine.Secret { + dst := []*engine.Secret{} + for k, v := range src { + if strings.TrimSpace(v.Secret) != "" { + dst = append(dst, &engine.Secret{ + Name: v.Secret, + Mask: true, + Env: k, + }) + } + } + return dst +} + +// helper function modifies the pipeline dependency graph to +// account for the clone step. +func configureCloneDeps(spec *engine.Spec) { + for _, step := range spec.Steps { + if step.Name == "clone" { + continue + } + if len(step.DependsOn) == 0 { + step.DependsOn = []string{"clone"} + } + } +} + +// helper function modifies the pipeline dependency graph to +// account for a disabled clone step. +func removeCloneDeps(spec *engine.Spec) { + for _, step := range spec.Steps { + if step.Name == "clone" { + return + } + } + for _, step := range spec.Steps { + if len(step.DependsOn) == 1 && + step.DependsOn[0] == "clone" { + step.DependsOn = []string{} + } + } +} + +// helper function modifies the pipeline dependency graph to +// account for the clone step. +func convertPullPolicy(s string) engine.PullPolicy { + switch strings.ToLower(s) { + case "always": + return engine.PullAlways + case "if-not-exists": + return engine.PullIfNotExists + case "never": + return engine.PullNever + default: + return engine.PullDefault + } +} diff --git a/engine/compiler/util_test.go b/engine/compiler/util_test.go new file mode 100644 index 0000000..ba9f3e8 --- /dev/null +++ b/engine/compiler/util_test.go @@ -0,0 +1,200 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "testing" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/runner-go/manifest" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func Test_isRunAlways(t *testing.T) { + step := new(resource.Step) + if isRunAlways(step) == true { + t.Errorf("Want always run false if empty when clause") + } + step.When.Status.Include = []string{"success"} + if isRunAlways(step) == true { + t.Errorf("Want always run false if when success") + } + step.When.Status.Include = []string{"failure"} + if isRunAlways(step) == true { + t.Errorf("Want always run false if when faiure") + } + step.When.Status.Include = []string{"success", "failure"} + if isRunAlways(step) == false { + t.Errorf("Want always run true if when success, failure") + } +} + +func Test_isRunOnFailure(t *testing.T) { + step := new(resource.Step) + if isRunOnFailure(step) == true { + t.Errorf("Want run on failure false if empty when clause") + } + step.When.Status.Include = []string{"success"} + if isRunOnFailure(step) == true { + t.Errorf("Want run on failure false if when success") + } + step.When.Status.Include = []string{"failure"} + if isRunOnFailure(step) == false { + t.Errorf("Want run on failure true if when faiure") + } + step.When.Status.Include = []string{"success", "failure"} + if isRunOnFailure(step) == false { + t.Errorf("Want run on failure true if when success, failure") + } +} + +func Test_isGraph(t *testing.T) { + spec := new(engine.Spec) + spec.Steps = []*engine.Step{ + {DependsOn: []string{}}, + } + if isGraph(spec) == true { + t.Errorf("Expect is graph false if deps not exist") + } + spec.Steps[0].DependsOn = []string{"clone"} + if isGraph(spec) == false { + t.Errorf("Expect is graph true if deps exist") + } +} + +func Test_configureSerial(t *testing.T) { + before := new(engine.Spec) + before.Steps = []*engine.Step{ + {Name: "build"}, + {Name: "test"}, + {Name: "deploy"}, + } + + after := new(engine.Spec) + after.Steps = []*engine.Step{ + {Name: "build"}, + {Name: "test", DependsOn: []string{"build"}}, + {Name: "deploy", DependsOn: []string{"test"}}, + } + configureSerial(before) + + opts := cmpopts.IgnoreUnexported(engine.Spec{}) + if diff := cmp.Diff(before, after, opts); diff != "" { + t.Errorf("Unexpected serial configuration") + t.Log(diff) + } +} + +func Test_convertStaticEnv(t *testing.T) { + vars := map[string]*manifest.Variable{ + "username": &manifest.Variable{Value: "octocat"}, + "password": &manifest.Variable{Secret: "password"}, + } + envs := convertStaticEnv(vars) + want := map[string]string{"username": "octocat"} + if diff := cmp.Diff(envs, want); diff != "" { + t.Errorf("Unexpected environment variable set") + t.Log(diff) + } +} + +func Test_convertSecretEnv(t *testing.T) { + vars := map[string]*manifest.Variable{ + "USERNAME": &manifest.Variable{Value: "octocat"}, + "PASSWORD": &manifest.Variable{Secret: "password"}, + } + envs := convertSecretEnv(vars) + want := []*engine.Secret{ + { + Name: "password", + Env: "PASSWORD", + Mask: true, + }, + } + if diff := cmp.Diff(envs, want); diff != "" { + t.Errorf("Unexpected secret list") + t.Log(diff) + } +} + +func Test_configureCloneDeps(t *testing.T) { + before := new(engine.Spec) + before.Steps = []*engine.Step{ + {Name: "clone"}, + {Name: "backend"}, + {Name: "frontend"}, + {Name: "deploy", DependsOn: []string{ + "backend", "frontend", + }}, + } + + after := new(engine.Spec) + after.Steps = []*engine.Step{ + {Name: "clone"}, + {Name: "backend", DependsOn: []string{"clone"}}, + {Name: "frontend", DependsOn: []string{"clone"}}, + {Name: "deploy", DependsOn: []string{ + "backend", "frontend", + }}, + } + configureCloneDeps(before) + + opts := cmpopts.IgnoreUnexported(engine.Spec{}) + if diff := cmp.Diff(before, after, opts); diff != "" { + t.Errorf("Unexpected dependency adjustment") + t.Log(diff) + } +} + +func Test_removeCloneDeps(t *testing.T) { + before := new(engine.Spec) + before.Steps = []*engine.Step{ + {Name: "backend", DependsOn: []string{"clone"}}, + {Name: "frontend", DependsOn: []string{"clone"}}, + {Name: "deploy", DependsOn: []string{ + "backend", "frontend", + }}, + } + + after := new(engine.Spec) + after.Steps = []*engine.Step{ + {Name: "backend", DependsOn: []string{}}, + {Name: "frontend", DependsOn: []string{}}, + {Name: "deploy", DependsOn: []string{ + "backend", "frontend", + }}, + } + removeCloneDeps(before) + + opts := cmpopts.IgnoreUnexported(engine.Spec{}) + if diff := cmp.Diff(before, after, opts); diff != "" { + t.Errorf("Unexpected result after removing clone deps") + t.Log(diff) + } +} + +func Test_removeCloneDeps_CloneEnabled(t *testing.T) { + before := new(engine.Spec) + before.Steps = []*engine.Step{ + {Name: "clone"}, + {Name: "test", DependsOn: []string{"clone"}}, + } + + after := new(engine.Spec) + after.Steps = []*engine.Step{ + {Name: "clone"}, + {Name: "test", DependsOn: []string{"clone"}}, + } + removeCloneDeps(before) + + opts := cmpopts.IgnoreUnexported(engine.Spec{}) + if diff := cmp.Diff(before, after, opts); diff != "" { + t.Errorf("Expect clone dependencies not removed") + t.Log(diff) + } +} diff --git a/engine/compiler/workspace.go b/engine/compiler/workspace.go new file mode 100644 index 0000000..0f369e9 --- /dev/null +++ b/engine/compiler/workspace.go @@ -0,0 +1,74 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + stdpath "path" + "strings" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" +) + +const ( + workspacePath = "/drone/src" + workspaceName = "workspace" + workspaceHostName = "host" +) + +func createWorkspace(from *resource.Pipeline) (base, path, full string) { + base = from.Workspace.Base + path = from.Workspace.Path + if base == "" { + base = workspacePath + } + full = stdpath.Join(base, path) + + if from.Platform.OS == "windows" { + base = toWindowsDrive(base) + path = toWindowsPath(path) + full = toWindowsDrive(full) + } + return base, path, full +} + +func setupWorkdir(src *resource.Step, dst *engine.Step, path string) { + // if the working directory is already set + // do not alter. + if dst.WorkingDir != "" { + return + } + // if the user is running the container as a + // service (detached mode) with no commands, we + // should use the default working directory. + if dst.Detach && len(src.Commands) == 0 { + return + } + // else set the working directory. + dst.WorkingDir = path +} + +// helper function appends the workspace base and +// path to the step's list of environment variables. +func setupWorkspaceEnv(step *engine.Step, base, path, full string) { + step.Envs["DRONE_WORKSPACE_BASE"] = base + step.Envs["DRONE_WORKSPACE_PATH"] = path + step.Envs["DRONE_WORKSPACE"] = full + step.Envs["CI_WORKSPACE_BASE"] = base + step.Envs["CI_WORKSPACE_PATH"] = path + step.Envs["CI_WORKSPACE"] = full +} + +// helper function converts the path to a valid windows +// path, including the default C drive. +func toWindowsDrive(s string) string { + return "c:" + toWindowsPath(s) +} + +// helper function converts the path to a valid windows +// path, replacing backslashes with forward slashes. +func toWindowsPath(s string) string { + return strings.Replace(s, "/", "\\", -1) +} diff --git a/engine/compiler/workspace_test.go b/engine/compiler/workspace_test.go new file mode 100644 index 0000000..cd4b909 --- /dev/null +++ b/engine/compiler/workspace_test.go @@ -0,0 +1,149 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package compiler + +import ( + "testing" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/runner-go/manifest" +) + +func TestSetupWorkspace(t *testing.T) { + tests := []struct { + path string + src *resource.Step + dst *engine.Step + want string + }{ + { + path: "/drone/src", + src: &resource.Step{}, + dst: &engine.Step{}, + want: "/drone/src", + }, + // do not override the user-defined working dir. + { + path: "/drone/src", + src: &resource.Step{}, + dst: &engine.Step{WorkingDir: "/foo"}, + want: "/foo", + }, + // do not override the default working directory + // for service containers with no commands. + { + path: "/drone/src", + src: &resource.Step{}, + dst: &engine.Step{Detach: true}, + want: "", + }, + // overrides the default working directory + // for service containers with commands. + { + path: "/drone/src", + src: &resource.Step{Commands: []string{"whoami"}}, + dst: &engine.Step{Detach: true}, + want: "/drone/src", + }, + } + for _, test := range tests { + setupWorkdir(test.src, test.dst, test.path) + if got, want := test.dst.WorkingDir, test.want; got != want { + t.Errorf("Want working_dir %s, got %s", want, got) + } + } +} + +func TestToWindows(t *testing.T) { + got := toWindowsDrive("/go/src/github.com/octocat/hello-world") + want := "c:\\go\\src\\github.com\\octocat\\hello-world" + if got != want { + t.Errorf("Want windows drive %q, got %q", want, got) + } +} + +func TestCreateWorkspace(t *testing.T) { + tests := []struct { + from *resource.Pipeline + base string + path string + full string + }{ + { + from: &resource.Pipeline{ + Workspace: resource.Workspace{ + Base: "", + Path: "", + }, + }, + base: "/drone/src", + path: "", + full: "/drone/src", + }, + { + from: &resource.Pipeline{ + Workspace: resource.Workspace{ + Base: "", + Path: "", + }, + Platform: manifest.Platform{ + OS: "windows", + }, + }, + base: "c:\\drone\\src", + path: "", + full: "c:\\drone\\src", + }, + { + from: &resource.Pipeline{ + Workspace: resource.Workspace{ + Base: "/drone", + Path: "src", + }, + }, + base: "/drone", + path: "src", + full: "/drone/src", + }, + { + from: &resource.Pipeline{ + Workspace: resource.Workspace{ + Base: "/drone", + Path: "src", + }, + Platform: manifest.Platform{ + OS: "windows", + }, + }, + base: "c:\\drone", + path: "src", + full: "c:\\drone\\src", + }, + { + from: &resource.Pipeline{ + Workspace: resource.Workspace{ + Base: "/foo", + Path: "bar", + }, + }, + base: "/foo", + path: "bar", + full: "/foo/bar", + }, + } + for _, test := range tests { + base, path, full := createWorkspace(test.from) + if got, want := test.base, base; got != want { + t.Errorf("Want workspace base %s, got %s", want, got) + } + if got, want := test.path, path; got != want { + t.Errorf("Want workspace path %s, got %s", want, got) + } + if got, want := test.full, full; got != want { + t.Errorf("Want workspace %s, got %s", want, got) + } + } +} diff --git a/engine/const.go b/engine/const.go new file mode 100644 index 0000000..36ec1da --- /dev/null +++ b/engine/const.go @@ -0,0 +1,118 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine + +import ( + "bytes" + "encoding/json" +) + +// PullPolicy defines the container image pull policy. +type PullPolicy int + +// PullPolicy enumeration. +const ( + PullDefault PullPolicy = iota + PullAlways + PullIfNotExists + PullNever +) + +func (p PullPolicy) String() string { + return pullPolicyID[p] +} + +var pullPolicyID = map[PullPolicy]string{ + PullDefault: "default", + PullAlways: "always", + PullIfNotExists: "if-not-exists", + PullNever: "never", +} + +var pullPolicyName = map[string]PullPolicy{ + "": PullDefault, + "default": PullDefault, + "always": PullAlways, + "if-not-exists": PullIfNotExists, + "never": PullNever, +} + +// MarshalJSON marshals the string representation of the +// pull type to JSON. +func (p *PullPolicy) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(pullPolicyID[*p]) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +// UnmarshalJSON unmarshals the json representation of the +// pull type from a string value. +func (p *PullPolicy) UnmarshalJSON(b []byte) error { + // unmarshal as string + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + // lookup value + *p = pullPolicyName[s] + return nil +} + +// RunPolicy defines the policy for starting containers +// based on the point-in-time pass or fail state of +// the pipeline. +type RunPolicy int + +// RunPolicy enumeration. +const ( + RunOnSuccess RunPolicy = iota + RunOnFailure + RunAlways + RunNever +) + +func (r RunPolicy) String() string { + return runPolicyID[r] +} + +var runPolicyID = map[RunPolicy]string{ + RunOnSuccess: "on-success", + RunOnFailure: "on-failure", + RunAlways: "always", + RunNever: "never", +} + +var runPolicyName = map[string]RunPolicy{ + "": RunOnSuccess, + "on-success": RunOnSuccess, + "on-failure": RunOnFailure, + "always": RunAlways, + "never": RunNever, +} + +// MarshalJSON marshals the string representation of the +// run type to JSON. +func (r *RunPolicy) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + buffer.WriteString(runPolicyID[*r]) + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +// UnmarshalJSON unmarshals the json representation of the +// run type from a string value. +func (r *RunPolicy) UnmarshalJSON(b []byte) error { + // unmarshal as string + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + // lookup value + *r = runPolicyName[s] + return nil +} diff --git a/engine/const_test.go b/engine/const_test.go new file mode 100644 index 0000000..b671470 --- /dev/null +++ b/engine/const_test.go @@ -0,0 +1,237 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine + +import ( + "bytes" + "encoding/json" + "testing" +) + +// +// runtime policy unit tests. +// + +func TestRunPolicy_Marshal(t *testing.T) { + tests := []struct { + policy RunPolicy + data string + }{ + { + policy: RunAlways, + data: `"always"`, + }, + { + policy: RunOnFailure, + data: `"on-failure"`, + }, + { + policy: RunOnSuccess, + data: `"on-success"`, + }, + { + policy: RunNever, + data: `"never"`, + }, + } + for _, test := range tests { + data, err := json.Marshal(&test.policy) + if err != nil { + t.Error(err) + return + } + if bytes.Equal([]byte(test.data), data) == false { + t.Errorf("Failed to marshal policy %s", test.policy) + } + } +} + +func TestRunPolicy_Unmarshal(t *testing.T) { + tests := []struct { + policy RunPolicy + data string + }{ + { + policy: RunAlways, + data: `"always"`, + }, + { + policy: RunOnFailure, + data: `"on-failure"`, + }, + { + policy: RunOnSuccess, + data: `"on-success"`, + }, + { + policy: RunNever, + data: `"never"`, + }, + { + // no policy should default to on-success + policy: RunOnSuccess, + data: `""`, + }, + } + for _, test := range tests { + var policy RunPolicy + err := json.Unmarshal([]byte(test.data), &policy) + if err != nil { + t.Error(err) + return + } + if got, want := policy, test.policy; got != want { + t.Errorf("Want policy %q, got %q", want, got) + } + } +} + +func TestRunPolicy_UnmarshalTypeError(t *testing.T) { + var policy RunPolicy + err := json.Unmarshal([]byte("[]"), &policy) + if _, ok := err.(*json.UnmarshalTypeError); !ok { + t.Errorf("Expect unmarshal error return when JSON invalid") + } +} + +func TestRunPolicy_String(t *testing.T) { + tests := []struct { + policy RunPolicy + value string + }{ + { + policy: RunAlways, + value: "always", + }, + { + policy: RunOnFailure, + value: "on-failure", + }, + { + policy: RunOnSuccess, + value: "on-success", + }, + } + for _, test := range tests { + if got, want := test.policy.String(), test.value; got != want { + t.Errorf("Want policy string %q, got %q", want, got) + } + } +} + +// +// pull policy unit tests. +// + +func TestPullPolicy_Marshal(t *testing.T) { + tests := []struct { + policy PullPolicy + data string + }{ + { + policy: PullAlways, + data: `"always"`, + }, + { + policy: PullDefault, + data: `"default"`, + }, + { + policy: PullIfNotExists, + data: `"if-not-exists"`, + }, + { + policy: PullNever, + data: `"never"`, + }, + } + for _, test := range tests { + data, err := json.Marshal(&test.policy) + if err != nil { + t.Error(err) + return + } + if bytes.Equal([]byte(test.data), data) == false { + t.Errorf("Failed to marshal policy %s", test.policy) + } + } +} + +func TestPullPolicy_Unmarshal(t *testing.T) { + tests := []struct { + policy PullPolicy + data string + }{ + { + policy: PullAlways, + data: `"always"`, + }, + { + policy: PullDefault, + data: `"default"`, + }, + { + policy: PullIfNotExists, + data: `"if-not-exists"`, + }, + { + policy: PullNever, + data: `"never"`, + }, + { + // no policy should default to on-success + policy: PullDefault, + data: `""`, + }, + } + for _, test := range tests { + var policy PullPolicy + err := json.Unmarshal([]byte(test.data), &policy) + if err != nil { + t.Error(err) + return + } + if got, want := policy, test.policy; got != want { + t.Errorf("Want policy %q, got %q", want, got) + } + } +} + +func TestPullPolicy_UnmarshalTypeError(t *testing.T) { + var policy PullPolicy + err := json.Unmarshal([]byte("[]"), &policy) + if _, ok := err.(*json.UnmarshalTypeError); !ok { + t.Errorf("Expect unmarshal error return when JSON invalid") + } +} + +func TestPullPolicy_String(t *testing.T) { + tests := []struct { + policy PullPolicy + value string + }{ + { + policy: PullAlways, + value: "always", + }, + { + policy: PullDefault, + value: "default", + }, + { + policy: PullIfNotExists, + value: "if-not-exists", + }, + { + policy: PullNever, + value: "never", + }, + } + for _, test := range tests { + if got, want := test.policy.String(), test.value; got != want { + t.Errorf("Want policy string %q, got %q", want, got) + } + } +} diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 0000000..02ec75d --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,23 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "io" +) + +// Engine is the interface that must be implemented by a +// pipeline execution engine. +type Engine interface { + // Setup the pipeline environment. + Setup(context.Context, *Spec) error + + // Destroy the pipeline environment. + Destroy(context.Context, *Spec) error + + // Run runs the pipeine step. + Run(context.Context, *Spec, *Step, io.Writer) (*State, error) +} diff --git a/engine/engine_impl.go b/engine/engine_impl.go new file mode 100644 index 0000000..b41805e --- /dev/null +++ b/engine/engine_impl.go @@ -0,0 +1,33 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "io" +) + +// New returns a new engine. +func New(publickeyFile, privatekeyFile string) (Engine, error) { + return &engine{}, nil +} + +type engine struct { +} + +// Setup the pipeline environment. +func (e *engine) Setup(ctx context.Context, spec *Spec) error { + return nil +} + +// Destroy the pipeline environment. +func (e *engine) Destroy(ctx context.Context, spec *Spec) error { + return nil +} + +// Run runs the pipeline step. +func (e *engine) Run(ctx context.Context, spec *Spec, step *Step, output io.Writer) (*State, error) { + return nil, nil +} diff --git a/engine/replacer/replacer.go b/engine/replacer/replacer.go new file mode 100644 index 0000000..3db9844 --- /dev/null +++ b/engine/replacer/replacer.go @@ -0,0 +1,56 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package replacer + +import ( + "fmt" + "io" + "strings" + + "github.com/drone-runners/drone-runner-docker/engine" +) + +const maskedf = "[secret:%s]" + +// Replacer is an io.Writer that finds and masks sensitive data. +type Replacer struct { + w io.WriteCloser + r *strings.Replacer +} + +// New returns a replacer that wraps writer w. +func New(w io.WriteCloser, secrets []*engine.Secret) io.WriteCloser { + var oldnew []string + for _, secret := range secrets { + if len(secret.Data) == 0 || secret.Mask == false { + continue + } + name := strings.ToLower(secret.Name) + masked := fmt.Sprintf(maskedf, name) + oldnew = append(oldnew, string(secret.Data)) + oldnew = append(oldnew, masked) + } + if len(oldnew) == 0 { + return w + } + return &Replacer{ + w: w, + r: strings.NewReplacer(oldnew...), + } +} + +// Write writes p to the base writer. The method scans for any +// sensitive data in p and masks before writing. +func (r *Replacer) Write(p []byte) (n int, err error) { + _, err = r.w.Write([]byte(r.r.Replace(string(p)))) + return len(p), err +} + +// Close closes the base writer. +func (r *Replacer) Close() error { + return r.w.Close() +} diff --git a/engine/replacer/replacer_test.go b/engine/replacer/replacer_test.go new file mode 100644 index 0000000..54c7462 --- /dev/null +++ b/engine/replacer/replacer_test.go @@ -0,0 +1,56 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package replacer + +import ( + "bytes" + "io" + "testing" + + "github.com/drone-runners/drone-runner-docker/engine" +) + +func TestReplace(t *testing.T) { + secrets := []*engine.Secret{ + {Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false}, + {Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: true}, + {Name: "DOCKER_EMAIL", Data: []byte(""), Mask: true}, + } + + buf := new(bytes.Buffer) + w := New(&nopCloser{buf}, secrets) + w.Write([]byte("username octocat password correct-horse-batter-staple")) + w.Close() + + if got, want := buf.String(), "username octocat password [secret:docker_password]"; got != want { + t.Errorf("Want masked string %s, got %s", want, got) + } +} + +// this test verifies that if there are no secrets to scan and +// mask, the io.WriteCloser is returned as-is. +func TestReplaceNone(t *testing.T) { + secrets := []*engine.Secret{ + {Name: "DOCKER_USERNAME", Data: []byte("octocat"), Mask: false}, + {Name: "DOCKER_PASSWORD", Data: []byte("correct-horse-batter-staple"), Mask: false}, + } + + buf := new(bytes.Buffer) + w := &nopCloser{buf} + r := New(w, secrets) + if w != r { + t.Errorf("Expect buffer returned with no replacer") + } +} + +type nopCloser struct { + io.Writer +} + +func (*nopCloser) Close() error { + return nil +} diff --git a/engine/resource/linter.go b/engine/resource/linter.go new file mode 100644 index 0000000..d9015db --- /dev/null +++ b/engine/resource/linter.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource diff --git a/engine/resource/lookup.go b/engine/resource/lookup.go new file mode 100644 index 0000000..75ad4b8 --- /dev/null +++ b/engine/resource/lookup.go @@ -0,0 +1,24 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import ( + "errors" + + "github.com/drone/runner-go/manifest" +) + +// Lookup returns the named pipeline from the Manifest. +func Lookup(name string, manifest *manifest.Manifest) (*Pipeline, error) { + for _, resource := range manifest.Resources { + if resource.GetName() != name { + continue + } + if pipeline, ok := resource.(*Pipeline); ok { + return pipeline, nil + } + } + return nil, errors.New("resource not found") +} diff --git a/engine/resource/lookup_test.go b/engine/resource/lookup_test.go new file mode 100644 index 0000000..894278a --- /dev/null +++ b/engine/resource/lookup_test.go @@ -0,0 +1,45 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import ( + "testing" + + "github.com/drone/runner-go/manifest" +) + +func TestLookup(t *testing.T) { + want := &Pipeline{Name: "default"} + m := &manifest.Manifest{ + Resources: []manifest.Resource{want}, + } + got, err := Lookup("default", m) + if err != nil { + t.Error(err) + } + if got != want { + t.Errorf("Expect resource not found error") + } +} + +func TestLookupNotFound(t *testing.T) { + m := &manifest.Manifest{ + Resources: []manifest.Resource{ + &manifest.Secret{ + Kind: "secret", + Name: "password", + }, + // matches name, but is not of kind pipeline + &manifest.Secret{ + Kind: "secret", + Name: "default", + }, + }, + } + _, err := Lookup("default", m) + if err == nil { + t.Errorf("Expect resource not found error") + } +} diff --git a/engine/resource/parser.go b/engine/resource/parser.go new file mode 100644 index 0000000..0dd4c85 --- /dev/null +++ b/engine/resource/parser.go @@ -0,0 +1,54 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import ( + "errors" + + "github.com/drone/runner-go/manifest" + + "github.com/buildkite/yaml" +) + +func init() { + manifest.Register(parse) +} + +// parse parses the raw resource and returns an Exec pipeline. +func parse(r *manifest.RawResource) (manifest.Resource, bool, error) { + if !match(r) { + return nil, false, nil + } + out := new(Pipeline) + err := yaml.Unmarshal(r.Data, out) + if err != nil { + return out, true, err + } + err = lint(out) + return out, true, err +} + +// match returns true if the resource matches the kind and type. +func match(r *manifest.RawResource) bool { + return r.Kind == Kind && r.Type == Type +} + +func lint(pipeline *Pipeline) error { + // ensure pipeline steps are not unique. + names := map[string]struct{}{} + for _, step := range pipeline.Steps { + if step.Name == "" { + return errors.New("Linter: invalid or missing step name") + } + if len(step.Name) > 100 { + return errors.New("Linter: step name cannot exceed 100 characters") + } + if _, ok := names[step.Name]; ok { + return errors.New("Linter: duplicate step name") + } + names[step.Name] = struct{}{} + } + return nil +} diff --git a/engine/resource/parser_test.go b/engine/resource/parser_test.go new file mode 100644 index 0000000..82cf43f --- /dev/null +++ b/engine/resource/parser_test.go @@ -0,0 +1,159 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import ( + "testing" + + "github.com/drone/runner-go/manifest" + + "github.com/google/go-cmp/cmp" +) + +func TestParse(t *testing.T) { + got, err := manifest.ParseFile("testdata/manifest.yml") + if err != nil { + t.Error(err) + return + } + + want := []manifest.Resource{ + &manifest.Signature{ + Kind: "signature", + Hmac: "a8842634682b78946a2", + }, + &manifest.Secret{ + Kind: "secret", + Type: "encrypted", + Name: "token", + Data: "f0e4c2f76c58916ec25", + }, + &Pipeline{ + Kind: "pipeline", + Type: "docker", + Name: "default", + Version: "1", + Workspace: Workspace{ + Path: "/drone/src", + }, + Platform: manifest.Platform{ + OS: "linux", + Arch: "arm64", + }, + Clone: manifest.Clone{ + Depth: 50, + }, + PullSecrets: []string{"dockerconfigjson"}, + Trigger: manifest.Conditions{ + Branch: manifest.Condition{ + Include: []string{"master"}, + }, + }, + Services: []*Step{ + { + Name: "redis", + Image: "redis:latest", + Entrypoint: []string{"/bin/redis-server"}, + Command: []string{"--debug"}, + }, + }, + Steps: []*Step{ + { + Name: "build", + Image: "golang", + Detach: false, + DependsOn: []string{"clone"}, + Commands: []string{ + "go build", + "go test", + }, + Environment: map[string]*manifest.Variable{ + "GOOS": &manifest.Variable{Value: "linux"}, + "GOARCH": &manifest.Variable{Value: "arm64"}, + }, + Failure: "ignore", + When: manifest.Conditions{ + Event: manifest.Condition{ + Include: []string{"push"}, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(got.Resources, want); diff != "" { + t.Errorf("Unexpected manifest") + t.Log(diff) + } +} + +func TestParseErr(t *testing.T) { + _, err := manifest.ParseFile("testdata/malformed.yml") + if err == nil { + t.Errorf("Expect error when malformed yaml") + } +} + +func TestParseLintErr(t *testing.T) { + _, err := manifest.ParseFile("testdata/linterr.yml") + if err == nil { + t.Errorf("Expect linter returns error") + return + } +} + +func TestParseNoMatch(t *testing.T) { + r := &manifest.RawResource{Kind: "pipeline", Type: "exec"} + _, match, _ := parse(r) + if match { + t.Errorf("Expect no match") + } +} + +func TestMatch(t *testing.T) { + r := &manifest.RawResource{ + Kind: "pipeline", + Type: "docker", + } + if match(r) == false { + t.Errorf("Expect match, got false") + } + + r = &manifest.RawResource{ + Kind: "approval", + Type: "docker", + } + if match(r) == true { + t.Errorf("Expect kind mismatch, got true") + } + + r = &manifest.RawResource{ + Kind: "pipeline", + Type: "dummy", + } + if match(r) == true { + t.Errorf("Expect type mismatch, got true") + } + +} + +func TestLint(t *testing.T) { + p := new(Pipeline) + p.Steps = []*Step{{Name: "build"}, {Name: "test"}} + if err := lint(p); err != nil { + t.Errorf("Expect no lint error, got %s", err) + } + + p.Steps = []*Step{{Name: "build"}, {Name: "build"}} + if err := lint(p); err == nil { + t.Errorf("Expect error when duplicate name") + } + + p.Steps = []*Step{{Name: "build"}, {Name: ""}} + if err := lint(p); err == nil { + t.Errorf("Expect error when empty name") + } +} diff --git a/engine/resource/pipeline.go b/engine/resource/pipeline.go new file mode 100644 index 0000000..7fd881e --- /dev/null +++ b/engine/resource/pipeline.go @@ -0,0 +1,153 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import "github.com/drone/runner-go/manifest" + +var ( + _ manifest.Resource = (*Pipeline)(nil) + _ manifest.TriggeredResource = (*Pipeline)(nil) + _ manifest.DependantResource = (*Pipeline)(nil) + _ manifest.PlatformResource = (*Pipeline)(nil) +) + +// TODO(bradrydzewski) add resource limits + +// Defines the Resource Kind and Type. +const ( + Kind = "pipeline" + Type = "docker" +) + +// Pipeline is a pipeline resource that executes pipelines +// on the host machine without any virtualization. +type Pipeline struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Deps []string `json:"depends_on,omitempty"` + + Clone manifest.Clone `json:"clone,omitempty"` + Concurrency manifest.Concurrency `json:"concurrency,omitempty"` + Node map[string]string `json:"node,omitempty"` + Platform manifest.Platform `json:"platform,omitempty"` + Trigger manifest.Conditions `json:"conditions,omitempty"` + + Services []*Step `json:"services,omitempty"` + Steps []*Step `json:"steps,omitempty"` + Volumes []*Volume `json:"volumes,omitempty"` + PullSecrets []string `json:"image_pull_secrets,omitempty" yaml:"image_pull_secrets"` + Workspace Workspace `json:"workspace,omitempty"` +} + +// GetVersion returns the resource version. +func (p *Pipeline) GetVersion() string { return p.Version } + +// GetKind returns the resource kind. +func (p *Pipeline) GetKind() string { return p.Kind } + +// GetType returns the resource type. +func (p *Pipeline) GetType() string { return p.Type } + +// GetName returns the resource name. +func (p *Pipeline) GetName() string { return p.Name } + +// GetDependsOn returns the resource dependencies. +func (p *Pipeline) GetDependsOn() []string { return p.Deps } + +// GetTrigger returns the resource triggers. +func (p *Pipeline) GetTrigger() manifest.Conditions { return p.Trigger } + +// GetNodes returns the resource node labels. +func (p *Pipeline) GetNodes() map[string]string { return p.Node } + +// GetPlatform returns the resource platform. +func (p *Pipeline) GetPlatform() manifest.Platform { return p.Platform } + +// GetConcurrency returns the resource concurrency limits. +func (p *Pipeline) GetConcurrency() manifest.Concurrency { return p.Concurrency } + +// GetStep returns the named step. If no step exists with the +// given name, a nil value is returned. +func (p *Pipeline) GetStep(name string) *Step { + for _, step := range p.Steps { + if step.Name == name { + return step + } + } + return nil +} + +type ( + // Step defines a Pipeline step. + Step struct { + Command []string `json:"command,omitempty"` + Commands []string `json:"commands,omitempty"` + Detach bool `json:"detach,omitempty"` + DependsOn []string `json:"depends_on,omitempty" yaml:"depends_on"` + Devices []*VolumeDevice `json:"devices,omitempty"` + DNS []string `json:"dns,omitempty"` + DNSSearch []string `json:"dns_search,omitempty" yaml:"dns_search"` + Entrypoint []string `json:"entrypoint,omitempty"` + Environment map[string]*manifest.Variable `json:"environment,omitempty"` + ExtraHosts []string `json:"extra_hosts,omitempty" yaml:"extra_hosts"` + Failure string `json:"failure,omitempty"` + Image string `json:"image,omitempty"` + Network string `json:"network_mode,omitempty" yaml:"network_mode"` + Name string `json:"name,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Pull string `json:"pull,omitempty"` + Settings map[string]*manifest.Parameter `json:"settings,omitempty"` + Shell string `json:"shell,omitempty"` + User string `json:"user,omitempty"` + Volumes []*VolumeMount `json:"volumes,omitempty"` + When manifest.Conditions `json:"when,omitempty"` + WorkingDir string `json:"working_dir,omitempty" yaml:"working_dir"` + + // Resources *Resources `json:"resources,omitempty"` + } + + // Volume that can be mounted by containers. + Volume struct { + Name string `json:"name,omitempty"` + EmptyDir *VolumeEmptyDir `json:"temp,omitempty" yaml:"temp"` + HostPath *VolumeHostPath `json:"host,omitempty" yaml:"host"` + } + + // VolumeDevice describes a mapping of a raw block + // device within a container. + VolumeDevice struct { + Name string `json:"name,omitempty"` + DevicePath string `json:"path,omitempty" yaml:"path"` + } + + // VolumeMount describes a mounting of a Volume + // within a container. + VolumeMount struct { + Name string `json:"name,omitempty"` + MountPath string `json:"path,omitempty" yaml:"path"` + } + + // VolumeEmptyDir mounts a temporary directory from the + // host node's filesystem into the container. This can + // be used as a shared scratch space. + VolumeEmptyDir struct { + Medium string `json:"medium,omitempty"` + SizeLimit manifest.BytesSize `json:"size_limit,omitempty" yaml:"size_limit"` + } + + // VolumeHostPath mounts a file or directory from the + // host node's filesystem into your container. + VolumeHostPath struct { + Path string `json:"path,omitempty"` + } + + // Workspace represents the pipeline workspace configuration. + Workspace struct { + Base string `json:"base,omitempty"` + Path string `json:"path,omitempty"` + } +) diff --git a/engine/resource/pipeline_test.go b/engine/resource/pipeline_test.go new file mode 100644 index 0000000..d6de414 --- /dev/null +++ b/engine/resource/pipeline_test.go @@ -0,0 +1,71 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package resource + +import ( + "testing" + + "github.com/drone/runner-go/manifest" + + "github.com/google/go-cmp/cmp" +) + +func TestGetStep(t *testing.T) { + step1 := &Step{Name: "build"} + step2 := &Step{Name: "test"} + pipeline := &Pipeline{ + Steps: []*Step{step1, step2}, + } + if pipeline.GetStep("build") != step1 { + t.Errorf("Expected named step") + } + if pipeline.GetStep("deploy") != nil { + t.Errorf("Expected nil step") + } +} + +func TestGetters(t *testing.T) { + platform := manifest.Platform{ + OS: "linux", + Arch: "amd64", + } + trigger := manifest.Conditions{ + Branch: manifest.Condition{ + Include: []string{"master"}, + }, + } + pipeline := &Pipeline{ + Version: "1.0.0", + Kind: "pipeline", + Type: "docker", + Name: "default", + Deps: []string{"before"}, + Platform: platform, + Trigger: trigger, + } + if got, want := pipeline.GetVersion(), pipeline.Version; got != want { + t.Errorf("Want Version %s, got %s", want, got) + } + if got, want := pipeline.GetKind(), pipeline.Kind; got != want { + t.Errorf("Want Kind %s, got %s", want, got) + } + if got, want := pipeline.GetType(), pipeline.Type; got != want { + t.Errorf("Want Type %s, got %s", want, got) + } + if got, want := pipeline.GetName(), pipeline.Name; got != want { + t.Errorf("Want Name %s, got %s", want, got) + } + if diff := cmp.Diff(pipeline.GetDependsOn(), pipeline.Deps); diff != "" { + t.Errorf("Unexpected Deps") + t.Log(diff) + } + if diff := cmp.Diff(pipeline.GetTrigger(), pipeline.Trigger); diff != "" { + t.Errorf("Unexpected Trigger") + t.Log(diff) + } + if got, want := pipeline.GetPlatform(), pipeline.Platform; got != want { + t.Errorf("Want Platform %s, got %s", want, got) + } +} diff --git a/engine/resource/testdata/linterr.yml b/engine/resource/testdata/linterr.yml new file mode 100644 index 0000000..bbe43ec --- /dev/null +++ b/engine/resource/testdata/linterr.yml @@ -0,0 +1,15 @@ +--- +kind: pipeline +type: docker + +server: + image: docker-18-04 + region: nyc1 + size: s-1vcpu-1gb + +steps: +- commands: + - go build + - go test + +... \ No newline at end of file diff --git a/engine/resource/testdata/malformed.yml b/engine/resource/testdata/malformed.yml new file mode 100644 index 0000000..7824f09 --- /dev/null +++ b/engine/resource/testdata/malformed.yml @@ -0,0 +1,8 @@ +--- +kind: pipeline +type: docker + +steps: + foo: bar + +... \ No newline at end of file diff --git a/engine/resource/testdata/manifest.yml b/engine/resource/testdata/manifest.yml new file mode 100644 index 0000000..c45f0e4 --- /dev/null +++ b/engine/resource/testdata/manifest.yml @@ -0,0 +1,54 @@ +--- +kind: signature +hmac: a8842634682b78946a2 + +--- +kind: secret +type: encrypted +name: token +data: f0e4c2f76c58916ec25 + +--- +kind: pipeline +type: docker +name: default +version: 1 + +platform: + os: linux + arch: arm64 + +workspace: + path: /drone/src + +clone: + depth: 50 + +steps: +- name: build + image: golang + detach: false + failure: ignore + commands: + - go build + - go test + environment: + GOOS: linux + GOARCH: arm64 + depends_on: [ clone ] + when: + event: [ push ] + +services: +- name: redis + image: redis:latest + entrypoint: [ "/bin/redis-server" ] + command: [ "--debug" ] + +image_pull_secrets: +- dockerconfigjson + +trigger: + branch: [ master ] + +... \ No newline at end of file diff --git a/engine/resource/testdata/nomatch.yml b/engine/resource/testdata/nomatch.yml new file mode 100644 index 0000000..7f1ef82 --- /dev/null +++ b/engine/resource/testdata/nomatch.yml @@ -0,0 +1,5 @@ +--- +kind: pipeline +type: docker + +... \ No newline at end of file diff --git a/engine/spec.go b/engine/spec.go new file mode 100644 index 0000000..0e5fa4c --- /dev/null +++ b/engine/spec.go @@ -0,0 +1,115 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine + +type ( + // Spec provides the pipeline spec. This provides the + // required instructions for reproducible pipeline + // execution. + Spec struct { + Platform Platform `json:"platform,omitempty"` + Files []*File `json:"files,omitempty"` + Steps []*Step `json:"steps,omitempty"` + Volumes []*Volume `json:"volumes,omitempty"` + Network Network `json:"network"` + } + + // Step defines a pipeline step. + Step struct { + ID string `json:"id,omitempty"` + Command []string `json:"args,omitempty"` + Detach bool `json:"detach,omitempty"` + DependsOn []string `json:"depends_on,omitempty"` + DNS []string `json:"dns,omitempty"` + DNSSearch []string `json:"dns_search,omitempty"` + Entrypoint []string `json:"entrypoint,omitempty"` + Envs map[string]string `json:"environment,omitempty"` + ExtraHosts []string `json:"extra_hosts,omitempty"` + Files []*File `json:"files,omitempty"` + IgnoreErr bool `json:"ignore_err,omitempty"` + IgnoreStdout bool `json:"ignore_stderr,omitempty"` + IgnoreStderr bool `json:"ignore_stdout,omitempty"` + Image string `json:"image,omitempty"` + Name string `json:"name,omitempty"` + Network string `json:"network,omitempty"` + Networks []string `json:"networks,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Pull PullPolicy `json:"pull,omitempty"` + RunPolicy RunPolicy `json:"run_policy,omitempty"` + Secrets []*Secret `json:"secrets,omitempty"` + User string `json:"user,omitempty"` + Volumes []*VolumeMount `json:"volumes,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + } + + // File defines a file that should be uploaded or + // mounted somewhere in the step container or virtual + // machine prior to command execution. + File struct { + Path string `json:"path,omitempty"` + Mode uint32 `json:"mode,omitempty"` + Data []byte `json:"data,omitempty"` + IsDir bool `json:"is_dir,omitempty"` + } + + // Platform defines the target platform. + Platform struct { + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + Variant string `json:"variant,omitempty"` + Version string `json:"version,omitempty"` + } + + // Secret represents a secret variable. + Secret struct { + Name string `json:"name,omitempty"` + Env string `json:"env,omitempty"` + Data []byte `json:"data,omitempty"` + Mask bool `json:"mask,omitempty"` + } + + // State represents the process state. + State struct { + ExitCode int // Container exit code + Exited bool // Container exited + OOMKilled bool // Container is oom killed + } + + // Volume that can be mounted by containers. + Volume struct { + EmptyDir *VolumeEmptyDir `json:"temp,omitempty"` + HostPath *VolumeHostPath `json:"host,omitempty"` + } + + // VolumeMount describes a mounting of a Volume + // within a container. + VolumeMount struct { + Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + } + + // VolumeEmptyDir mounts a temporary directory from the + // host node's filesystem into the container. This can + // be used as a shared scratch space. + VolumeEmptyDir struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Medium string `json:"medium,omitempty"` + SizeLimit int64 `json:"size_limit,omitempty"` + } + + // VolumeHostPath mounts a file or directory from the + // host node's filesystem into your container. + VolumeHostPath struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + } + + // Network that is created and attached to containers + Network struct { + ID string `json:"id,omitempty"` + } +) diff --git a/engine/util.go b/engine/util.go new file mode 100644 index 0000000..e0565c4 --- /dev/null +++ b/engine/util.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine diff --git a/engine/util_test.go b/engine/util_test.go new file mode 100644 index 0000000..e0565c4 --- /dev/null +++ b/engine/util_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package engine diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7ee7e24 --- /dev/null +++ b/go.mod @@ -0,0 +1,33 @@ +module github.com/drone-runners/drone-runner-docker + +go 1.12 + +require ( + github.com/buildkite/yaml v2.1.0+incompatible + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 + github.com/digitalocean/godo v1.19.0 + github.com/docker/distribution v2.7.1+incompatible + github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba + github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1 + github.com/drone/drone-yaml v1.2.2 + github.com/drone/envsubst v1.0.2 + github.com/drone/runner-go v1.2.3-0.20191008184914-35b0486b7f23 + github.com/drone/signal v1.0.0 + github.com/ghodss/yaml v1.0.0 + github.com/google/go-cmp v0.3.0 + github.com/gosimple/slug v1.5.0 + github.com/hashicorp/go-multierror v1.0.0 + github.com/joho/godotenv v1.3.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-isatty v0.0.8 + github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 + github.com/pkg/errors v0.8.1 // indirect + github.com/pkg/sftp v1.10.1-0.20190613163056-79ae07e7783e + github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect + github.com/sirupsen/logrus v1.4.2 + golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + gopkg.in/alecthomas/kingpin.v2 v2.2.6 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8d778be --- /dev/null +++ b/go.sum @@ -0,0 +1,160 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY= +github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d h1:j6oB/WPCigdOkxtuPl1VSIiLpy7Mdsu6phQffbF19Ng= +github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs= +github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc= +github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= +github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/digitalocean/godo v1.19.0 h1:9ApuchfzGD/XI8Zm0RRnZnytdfYHPjPTRKTnmzQNV7o= +github.com/digitalocean/godo v1.19.0/go.mod h1:AAPQ+tiM4st79QHlEBTg8LM7JQNre4SAQCbn56wEyKY= +github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/drone/drone-go v1.0.4 h1:Yom1lix1Lmk3KmKIsBSQJF1bw0YR2lDGaFQrXxqHMko= +github.com/drone/drone-go v1.0.4/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4= +github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba h1:GKiT4UPBligLXJAP1zRllHvTUygAAlgS3t9LM9aasp0= +github.com/drone/drone-go v1.0.5-0.20190504210458-4d6116b897ba/go.mod h1:GxyeGClYohaKNYJv/ZpsmVHtMJ7WhoT+uDaJNcDIrk4= +github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1 h1:9xaZM1rM1/0FqFEijgnFcvWd0vRqOw+iO1YR7pBgPCw= +github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs= +github.com/drone/drone-yaml v1.2.2 h1:Srf8OlAHhR7SXX5Ax01dP5tpZENsrEKyg35E2nNkIew= +github.com/drone/drone-yaml v1.2.2/go.mod h1:QsqliFK8nG04AHFN9tTn9XJomRBQHD4wcejWW1uz/10= +github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo= +github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0= +github.com/drone/runner-go v1.1.0 h1:x2UEG8POjACkN28tqyHOEffZ4V4XUjea95lSbzx+xPY= +github.com/drone/runner-go v1.1.0/go.mod h1:+XIpVivm7KINzTD1oabOy+IwBKNBExl0Sqwhlh8E7/0= +github.com/drone/runner-go v1.1.1-0.20190715182255-1c863ab7b042 h1:KAuPeK2bf0UMA/PegB4LLrzqwuHIGB2fzFeUf8pygwM= +github.com/drone/runner-go v1.1.1-0.20190715182255-1c863ab7b042/go.mod h1:GvB5hX023g0n5UZUjMBmudk6givdDDuLcls7Nolm5v8= +github.com/drone/runner-go v1.1.1-0.20190715221101-a8307ea13bc0 h1:9NFdh0aIbP9ls4/GCWW+YwRb/PaQUQxrhfxfNOt4XEw= +github.com/drone/runner-go v1.1.1-0.20190715221101-a8307ea13bc0/go.mod h1:GvB5hX023g0n5UZUjMBmudk6givdDDuLcls7Nolm5v8= +github.com/drone/runner-go v1.1.1-0.20190716043234-8449fe8013cd h1:DKxgLLjdzPUNuJVpLUIndY2amxR1ciLxqIXaoy4/SYA= +github.com/drone/runner-go v1.1.1-0.20190716043234-8449fe8013cd/go.mod h1:GvB5hX023g0n5UZUjMBmudk6givdDDuLcls7Nolm5v8= +github.com/drone/runner-go v1.1.1-0.20190716194717-969b4e42cd4b h1:PWQTdNZujrBy1ESeAPa2afKSvWjD/YfCLQgsaAz4drA= +github.com/drone/runner-go v1.1.1-0.20190716194717-969b4e42cd4b/go.mod h1:GvB5hX023g0n5UZUjMBmudk6givdDDuLcls7Nolm5v8= +github.com/drone/runner-go v1.2.2 h1:fwYgjyJl6KdjQGEUFof9+HLtNpK3iHq7UuR+/aYNyDk= +github.com/drone/runner-go v1.2.3-0.20191008184914-35b0486b7f23 h1:sj9myNBjsR7C6HFdR0Lp79KtZw5MHf97FfRIsmoA1Ys= +github.com/drone/runner-go v1.2.3-0.20191008184914-35b0486b7f23/go.mod h1:GvB5hX023g0n5UZUjMBmudk6givdDDuLcls7Nolm5v8= +github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI= +github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gosimple/slug v1.5.0 h1:AIIjgCjHcLpX8LzM2NpG4QGW9kUfqv0OLiFRfPv/H3E= +github.com/gosimple/slug v1.5.0/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 h1:dnMxwus89s86tI8rcGVp2HwZzlz7c5o92VOy7dSckBQ= +github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4/go.mod h1:cojhOHk1gbMeklOyDP2oKKLftefXoJreOQGOrXk+Z38= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.0 h1:DGA1KlA9esU6WcicH+P8PxFZOl15O6GYtab1cIJdOlE= +github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= +github.com/pkg/sftp v1.10.1-0.20190613163056-79ae07e7783e h1:OFJvqBwYiN41kBIfsgm7DZhqT3bMljQ4UIr86BcPoVI= +github.com/pkg/sftp v1.10.1-0.20190613163056-79ae07e7783e/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ= +github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/internal/internal.go b/internal/internal.go new file mode 100644 index 0000000..734adf1 --- /dev/null +++ b/internal/internal.go @@ -0,0 +1,6 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +// Package internal contains runner internals. +package internal diff --git a/internal/match/match.go b/internal/match/match.go new file mode 100644 index 0000000..fb282a8 --- /dev/null +++ b/internal/match/match.go @@ -0,0 +1,51 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package match + +import ( + "path/filepath" + + "github.com/drone/drone-go/drone" +) + +// NOTE most runners do not require match capabilities. This is +// provided as a defense in depth mechanism given the sensitive +// nature of this runner executing code directly on the host. +// The matching function is a last line of defence to prevent +// unauthorized code from running on the host machine. + +// Func returns a new match function that returns true if the +// repository and build do not match the allowd repository names +// and build events. +func Func(repos, events []string, trusted bool) func(*drone.Repo, *drone.Build) bool { + return func(repo *drone.Repo, build *drone.Build) bool { + // if trusted mode is enabled, only match repositories + // that are trusted. + if trusted && repo.Trusted == false { + return false + } + if match(repo.Slug, repos) == false { + return false + } + if match(build.Event, events) == false { + return false + } + return true + } +} + +func match(s string, patterns []string) bool { + // if no matching patterns are defined the string + // is always considered a match. + if len(patterns) == 0 { + return true + } + for _, pattern := range patterns { + if match, _ := filepath.Match(pattern, s); match { + return true + } + } + return false +} diff --git a/internal/match/match_test.go b/internal/match/match_test.go new file mode 100644 index 0000000..0c477bf --- /dev/null +++ b/internal/match/match_test.go @@ -0,0 +1,125 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package match + +import ( + "testing" + + "github.com/drone/drone-go/drone" +) + +func TestFunc(t *testing.T) { + tests := []struct { + repo string + event string + trusted bool + match bool + matcher func(*drone.Repo, *drone.Build) bool + }{ + // + // Expect match true + // + + // repository, event and trusted flag matching + { + repo: "octocat/hello-world", + event: "push", + trusted: true, + match: true, + matcher: Func([]string{"spaceghost/*", "octocat/*"}, []string{"push"}, true), + }, + // repoisitory matching + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: false, + match: true, + matcher: Func([]string{"spaceghost/*", "octocat/*"}, []string{}, false), + }, + // event matching + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: false, + match: true, + matcher: Func([]string{}, []string{"pull_request"}, false), + }, + // trusted flag matching + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: true, + match: true, + matcher: Func([]string{}, []string{}, true), + }, + + // + // Expect match false + // + + // repository matching + { + repo: "spaceghost/hello-world", + event: "pull_request", + trusted: false, + match: false, + matcher: Func([]string{"octocat/*"}, []string{}, false), + }, + // event matching + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: false, + match: false, + matcher: Func([]string{}, []string{"push"}, false), + }, + // trusted flag matching + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: false, + match: false, + matcher: Func([]string{}, []string{}, true), + }, + // does not match repository + { + repo: "foo/hello-world", + event: "push", + trusted: true, + match: false, + matcher: Func([]string{"spaceghost/*", "octocat/*"}, []string{"push"}, true), + }, + // does not match event + { + repo: "octocat/hello-world", + event: "pull_request", + trusted: true, + match: false, + matcher: Func([]string{"spaceghost/*", "octocat/*"}, []string{"push"}, true), + }, + // does not match trusted flag + { + repo: "octocat/hello-world", + event: "push", + trusted: false, + match: false, + matcher: Func([]string{"spaceghost/*", "octocat/*"}, []string{"push"}, true), + }, + } + + for i, test := range tests { + repo := &drone.Repo{ + Slug: test.repo, + Trusted: test.trusted, + } + build := &drone.Build{ + Event: test.event, + } + match := test.matcher(repo, build) + if match != test.match { + t.Errorf("Expect match %v at index %d", test.match, i) + } + } +} diff --git a/internal/mock/mock.go b/internal/mock/mock.go new file mode 100644 index 0000000..eb963b4 --- /dev/null +++ b/internal/mock/mock.go @@ -0,0 +1,8 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package mock + +//go:generate mockgen -package=mock -destination=mock_engine_gen.go github.com/drone-runners/drone-runner-docker/engine Engine +//go:generate mockgen -package=mock -destination=mock_execer_gen.go github.com/drone-runners/drone-runner-docker/runtime Execer diff --git a/internal/platform/platform.go b/internal/platform/platform.go new file mode 100644 index 0000000..0cbb11c --- /dev/null +++ b/internal/platform/platform.go @@ -0,0 +1,177 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +// Package platform contains code to provision and destroy server +// instances on the Digital Ocean cloud platform. +package platform + +import ( + "context" + "time" + + "github.com/drone/runner-go/logger" + + "github.com/digitalocean/godo" + "golang.org/x/oauth2" +) + +type ( + // RegisterArgs provides arguments to register the SSH + // public key with the account. + RegisterArgs struct { + Fingerprint string + Name string + Data string + Token string + } + + // DestroyArgs provides arguments to destroy the server + // instance. + DestroyArgs struct { + ID int + IP string + Token string + } + + // ProvisionArgs provides arguments to provision instances. + ProvisionArgs struct { + Key string + Image string + Name string + Region string + Size string + Token string + } + + // Instance represents a provisioned server instance. + Instance struct { + ID int + IP string + } +) + +// Provision provisions the server instance. +func Provision(ctx context.Context, args ProvisionArgs) (Instance, error) { + res := Instance{} + req := &godo.DropletCreateRequest{ + Name: args.Name, + Region: args.Region, + Size: args.Size, + Tags: []string{"drone"}, + IPv6: false, + SSHKeys: []godo.DropletCreateSSHKey{ + {Fingerprint: args.Key}, + }, + Image: godo.DropletCreateImage{ + Slug: args.Image, + }, + } + + logger := logger.FromContext(ctx). + WithField("region", req.Region). + WithField("image", req.Image.Slug). + WithField("size", req.Size). + WithField("name", req.Name) + + logger.Debug("instance create") + + client := newClient(ctx, args.Token) + droplet, _, err := client.Droplets.Create(ctx, req) + if err != nil { + logger.WithError(err).Error("cannot create instance") + return res, err + } + + // record the droplet ID + res.ID = droplet.ID + + logger.WithField("name", req.Name). + Info("instance created") + + // poll the digitalocean endpoint for server updates + // and exit when a network address is allocated. + interval := time.Duration(0) +poller: + for { + select { + case <-ctx.Done(): + logger.WithField("name", req.Name). + Debug("cannot ascertain network") + + return res, ctx.Err() + case <-time.After(interval): + interval = time.Second * 30 + + logger.WithField("name", req.Name). + Debug("find instance network") + + droplet, _, err = client.Droplets.Get(ctx, res.ID) + if err != nil { + logger.WithError(err). + Error("cannot find instance") + return res, err + } + + for _, network := range droplet.Networks.V4 { + if network.Type == "public" { + res.IP = network.IPAddress + } + } + + if res.IP != "" { + break poller + } + } + } + + logger.WithField("name", req.Name). + WithField("ip", res.IP). + WithField("id", res.ID). + Debug("instance network ready") + + return res, nil +} + +// Destroy destroys the server instance. +func Destroy(ctx context.Context, args DestroyArgs) error { + client := newClient(ctx, args.Token) + _, err := client.Droplets.Delete(ctx, args.ID) + if err != nil { + logger.FromContext(ctx). + WithError(err). + WithField("id", args.ID). + WithField("ip", args.IP). + Error("cannot terminate server") + } + return err +} + +// RegisterKey registers the ssh public key with the account if +// it is not already registered. +func RegisterKey(ctx context.Context, args RegisterArgs) error { + client := newClient(ctx, args.Token) + _, _, err := client.Keys.GetByFingerprint(ctx, args.Fingerprint) + if err == nil { + return nil + } + + // if the ssh key does not exists we attempt to register + // with the digital ocean account. + _, _, err = client.Keys.Create(ctx, &godo.KeyCreateRequest{ + Name: args.Name, + PublicKey: args.Data, + }) + return err +} + +// helper function returns a new docker client. +func newClient(ctx context.Context, token string) *godo.Client { + return godo.NewClient( + oauth2.NewClient(ctx, oauth2.StaticTokenSource( + &oauth2.Token{ + AccessToken: token, + }, + )), + ) +} diff --git a/internal/platform/platform_test.go b/internal/platform/platform_test.go new file mode 100644 index 0000000..ce9b1a7 --- /dev/null +++ b/internal/platform/platform_test.go @@ -0,0 +1,5 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package platform diff --git a/licenses/Polyform-Free-Trial.md b/licenses/Polyform-Free-Trial.md new file mode 100644 index 0000000..78e8552 --- /dev/null +++ b/licenses/Polyform-Free-Trial.md @@ -0,0 +1,93 @@ +# Polyform Free Trial License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the software +to do everything you might do with the software that would +otherwise infringe the licensor's copyright in it for any +permitted purpose. However, you may only make changes or +new works based on the software according to [Changes and New +Works License](#changes-and-new-works-license), and you may +not distribute copies of the software. + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## Free Trial + +Use to evaluate whether the software suits a particular +application for less than 32 consecutive calendar days, on +behalf of you or your company, is use for a permitted purpose. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +If you violate any of these terms, or do anything with the +software not covered by your licenses, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. \ No newline at end of file diff --git a/licenses/Polyform-Small-Business.md b/licenses/Polyform-Small-Business.md new file mode 100644 index 0000000..ec0d725 --- /dev/null +++ b/licenses/Polyform-Small-Business.md @@ -0,0 +1,121 @@ +# Polyform Small Business License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the +software to do everything you might do with the software +that would otherwise infringe the licensor's copyright +in it for any permitted purpose. However, you may +only distribute the software according to [Distribution +License](#distribution-license) and make changes or new works +based on the software according to [Changes and New Works +License](#changes-and-new-works-license). + +## Distribution License + +The licensor grants you an additional copyright license +to distribute copies of the software. Your license +to distribute covers distributing the software with +changes and new works permitted by [Changes and New Works +License](#changes-and-new-works-license). + +## Notices + +You must ensure that anyone who gets a copy of any part of +the software from you also gets a copy of these terms or the +URL for them above, as well as copies of any plain-text lines +beginning with `Required Notice:` that the licensor provided +with the software. For example: + +> Required Notice: Copyright Yoyodyne, Inc. (http://example.com) + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## Small Business + +Use of the software for the benefit of your company is use for +a permitted purpose if your company has fewer than 100 total +individuals working as employees and independent contractors, +and less than 1,000,000 USD (2019) total revenue in the prior +tax year. Adjust this revenue threshold for inflation according +to the United States Bureau of Labor Statistics' consumer price +index for all urban consumers, U.S. city average, for all items, +not seasonally adjusted, with 1982–1984=100 reference base. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. diff --git a/main.go b/main.go new file mode 100644 index 0000000..2173908 --- /dev/null +++ b/main.go @@ -0,0 +1,14 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package main + +import ( + "github.com/drone-runners/drone-runner-docker/command" + _ "github.com/joho/godotenv/autoload" +) + +func main() { + command.Command() +} diff --git a/runtime/execer.go b/runtime/execer.go new file mode 100644 index 0000000..9071abf --- /dev/null +++ b/runtime/execer.go @@ -0,0 +1,242 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime + +import ( + "context" + "sync" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/replacer" + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/logger" + "github.com/drone/runner-go/pipeline" + + "github.com/hashicorp/go-multierror" + "github.com/natessilva/dag" + "golang.org/x/sync/semaphore" +) + +// Execer is the execution context for executing the intermediate +// representation of a pipeline. +type Execer interface { + Exec(context.Context, *engine.Spec, *pipeline.State) error +} + +type execer struct { + mu sync.Mutex + engine engine.Engine + reporter pipeline.Reporter + streamer pipeline.Streamer + sem *semaphore.Weighted +} + +// NewExecer returns a new execer used +func NewExecer( + reporter pipeline.Reporter, + streamer pipeline.Streamer, + engine engine.Engine, + procs int64, +) Execer { + exec := &execer{ + reporter: reporter, + streamer: streamer, + engine: engine, + } + if procs > 0 { + // optional semaphor that limits the number of steps + // that can execute concurrently. + exec.sem = semaphore.NewWeighted(procs) + } + return exec +} + +// Exec executes the intermediate representation of the pipeline +// and returns an error if execution fails. +func (e *execer) Exec(ctx context.Context, spec *engine.Spec, state *pipeline.State) error { + defer e.engine.Destroy(noContext, spec) + + if err := e.engine.Setup(noContext, spec); err != nil { + state.FailAll(err) + return e.reporter.ReportStage(noContext, state) + } + + // create a directed graph, where each vertex in the graph + // is a pipeline step. + var d dag.Runner + for _, s := range spec.Steps { + step := s + d.AddVertex(step.Name, func() error { + return e.exec(ctx, state, spec, step) + }) + } + + // create the vertex edges from the values configured in the + // depends_on attribute. + for _, s := range spec.Steps { + for _, dep := range s.DependsOn { + d.AddEdge(dep, s.Name) + } + } + + var result error + if err := d.Run(); err != nil { + multierror.Append(result, err) + } + + // once pipeline execution completes, notify the state + // manageer that all steps are finished. + state.FinishAll() + if err := e.reporter.ReportStage(noContext, state); err != nil { + multierror.Append(result, err) + } + return result +} + +func (e *execer) exec(ctx context.Context, state *pipeline.State, spec *engine.Spec, step *engine.Step) error { + var result error + + select { + case <-ctx.Done(): + state.Cancel() + return nil + default: + } + + log := logger.FromContext(ctx) + log = log.WithField("step.name", step.Name) + ctx = logger.WithContext(ctx, log) + + if e.sem != nil { + // the semaphore limits the number of steps that can run + // concurrently. acquire the semaphore and release when + // the pipeline completes. + if err := e.sem.Acquire(ctx, 1); err != nil { + return nil + } + + defer func() { + // recover from a panic to ensure the semaphore is + // released to prevent deadlock. we do not expect a + // panic, however, we are being overly cautious. + if r := recover(); r != nil { + // TODO(bradrydzewsi) log the panic. + } + // release the semaphore + e.sem.Release(1) + }() + } + + switch { + case state.Skipped(): + return nil + case state.Cancelled(): + return nil + case step.RunPolicy == engine.RunNever: + return nil + case step.RunPolicy == engine.RunAlways: + break + case step.RunPolicy == engine.RunOnFailure && state.Failed() == false: + state.Skip(step.Name) + return e.reporter.ReportStep(noContext, state, step.Name) + case step.RunPolicy == engine.RunOnSuccess && state.Failed(): + state.Skip(step.Name) + return e.reporter.ReportStep(noContext, state, step.Name) + } + + state.Start(step.Name) + err := e.reporter.ReportStep(noContext, state, step.Name) + if err != nil { + return err + } + + copy := cloneStep(step) + + // the pipeline environment variables need to be updated to + // reflect the current state of the build and stage. + state.Lock() + copy.Envs = environ.Combine( + copy.Envs, + environ.Build(state.Build), + environ.Stage(state.Stage), + environ.Step(findStep(state, step.Name)), + ) + state.Unlock() + + // writer used to stream build logs. + wc := e.streamer.Stream(noContext, state, step.Name) + wc = replacer.New(wc, step.Secrets) + + // if the step is configured as a daemon, it is detached + // from the main process and executed separately. + // todo(bradrydzewski) this code is still experimental. + if step.Detach { + go func() { + e.engine.Run(ctx, spec, copy, wc) + wc.Close() + }() + return nil + } + + exited, err := e.engine.Run(ctx, spec, copy, wc) + + // close the stream. If the session is a remote session, the + // full log buffer is uploaded to the remote server. + if err := wc.Close(); err != nil { + multierror.Append(result, err) + } + + if exited != nil { + state.Finish(step.Name, exited.ExitCode) + err := e.reporter.ReportStep(noContext, state, step.Name) + if err != nil { + multierror.Append(result, err) + } + // if the exit code is 78 the system will skip all + // subsequent pending steps in the pipeline. + if exited.ExitCode == 78 { + state.SkipAll() + } + return result + } + + switch err { + case context.Canceled, context.DeadlineExceeded: + state.Cancel() + return nil + } + + // if the step failed with an internal error (as oppsed to a + // runtime error) the step is failed. + state.Fail(step.Name, err) + err = e.reporter.ReportStep(noContext, state, step.Name) + if err != nil { + multierror.Append(result, err) + } + return result +} + +// helper function to clone a step. The runner mutates a step to +// update the environment variables to reflect the current +// pipeline state. +func cloneStep(src *engine.Step) *engine.Step { + dst := new(engine.Step) + *dst = *src + dst.Envs = environ.Combine(src.Envs) + return dst +} + +// helper function returns the named step from the state. +func findStep(state *pipeline.State, name string) *drone.Step { + for _, step := range state.Stage.Steps { + if step.Name == name { + return step + } + } + panic("step not found: " + name) +} diff --git a/runtime/execer_test.go b/runtime/execer_test.go new file mode 100644 index 0000000..784ab5c --- /dev/null +++ b/runtime/execer_test.go @@ -0,0 +1,39 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime + +import ( + "testing" +) + +func TestExec(t *testing.T) { + t.Skip() +} + +func TestExec_NonZeroExit(t *testing.T) { + t.Skip() +} + +func TestExec_Exit78(t *testing.T) { + t.Skip() +} + +func TestExec_Error(t *testing.T) { + t.Skip() +} + +func TestExec_CtxError(t *testing.T) { + t.Skip() +} + +func TestExec_ReportError(t *testing.T) { + t.Skip() +} + +func TestExec_SkipCtxDone(t *testing.T) { + t.Skip() +} diff --git a/runtime/poller.go b/runtime/poller.go new file mode 100644 index 0000000..4775d25 --- /dev/null +++ b/runtime/poller.go @@ -0,0 +1,72 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime + +import ( + "context" + "sync" + + "github.com/drone/runner-go/client" + "github.com/drone/runner-go/logger" +) + +var noContext = context.Background() + +// Poller polls the server for pending stages and dispatches +// for execution by the Runner. +type Poller struct { + Client client.Client + Filter *client.Filter + Runner *Runner +} + +// Poll opens N connections to the server to poll for pending +// stages for execution. Pending stages are dispatched to a +// Runner for execution. +func (p *Poller) Poll(ctx context.Context, n int) { + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + for { + select { + case <-ctx.Done(): + wg.Done() + return + default: + p.poll(ctx, i+1) + } + } + }(i) + } + + wg.Wait() +} + +// poll requests a stage for execution from the server, and then +// dispatches for execution. +func (p *Poller) poll(ctx context.Context, thread int) error { + log := logger.FromContext(ctx).WithField("thread", thread) + log.WithField("thread", thread).Debug("request stage from remote server") + + // request a new build stage for execution from the central + // build server. + stage, err := p.Client.Request(ctx, p.Filter) + if err != nil { + log.WithError(err).Error("cannot request stage") + return err + } + + // exit if a nil or empty stage is returned from the system + // and allow the runner to retry. + if stage == nil || stage.ID == 0 { + return nil + } + + return p.Runner.Run( + logger.WithContext(noContext, log), stage) +} diff --git a/runtime/poller_test.go b/runtime/poller_test.go new file mode 100644 index 0000000..519a535 --- /dev/null +++ b/runtime/poller_test.go @@ -0,0 +1,27 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime + +import ( + "testing" +) + +func TestPoll(t *testing.T) { + t.Skip() +} + +func TestPoll_NilStage(t *testing.T) { + t.Skip() +} + +func TestPoll_EmptyStage(t *testing.T) { + t.Skip() +} + +func TestPoll_RequestError(t *testing.T) { + t.Skip() +} diff --git a/runtime/runner.go b/runtime/runner.go new file mode 100644 index 0000000..abab7ab --- /dev/null +++ b/runtime/runner.go @@ -0,0 +1,236 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/compiler" + "github.com/drone-runners/drone-runner-docker/engine/resource" + + "github.com/drone/drone-go/drone" + "github.com/drone/envsubst" + "github.com/drone/runner-go/client" + "github.com/drone/runner-go/environ" + "github.com/drone/runner-go/logger" + "github.com/drone/runner-go/manifest" + "github.com/drone/runner-go/pipeline" + "github.com/drone/runner-go/secret" +) + +// Runnner runs the pipeline. +type Runner struct { + // Client is the remote client responsible for interacting + // with the central server. + Client client.Client + + // Execer is responsible for executing intermediate + // representation of the pipeline and returns its results. + Execer Execer + + // Reporter reports pipeline status back to the remote + // server. + Reporter pipeline.Reporter + + // Environ provides custom, global environment variables + // that are added to every pipeline step. + Environ map[string]string + + // Machine provides the runner with the name of the host + // machine executing the pipeline. + Machine string + + // Match is an optional function that returns true if the + // repository or build match user-defined criteria. This is + // intended as a security measure to prevent a runner from + // processing an unwanted pipeline. + Match func(*drone.Repo, *drone.Build) bool + + // Secret provides the compiler with secrets. + Secret secret.Provider +} + +// Run runs the pipeline stage. +func (s *Runner) Run(ctx context.Context, stage *drone.Stage) error { + log := logger.FromContext(ctx). + WithField("stage.id", stage.ID). + WithField("stage.name", stage.Name). + WithField("stage.number", stage.Number) + + log.Debug("stage received") + + // delivery to a single agent is not guaranteed, which means + // we need confirm receipt. The first agent that confirms + // receipt of the stage can assume ownership. + + stage.Machine = s.Machine + err := s.Client.Accept(ctx, stage) + if err != nil { + log.WithError(err).Error("cannot accept stage") + return err + } + + log.Debug("stage accepted") + + data, err := s.Client.Detail(ctx, stage) + if err != nil { + log.WithError(err).Error("cannot get stage details") + return err + } + + log = log.WithField("repo.id", data.Repo.ID). + WithField("repo.namespace", data.Repo.Namespace). + WithField("repo.name", data.Repo.Name). + WithField("build.id", data.Build.ID). + WithField("build.number", data.Build.Number) + + log.Debug("stage details fetched") + + ctxdone, cancel := context.WithCancel(ctx) + defer cancel() + + timeout := time.Duration(data.Repo.Timeout) * time.Minute + ctxtimeout, cancel := context.WithTimeout(ctxdone, timeout) + defer cancel() + + ctxcancel, cancel := context.WithCancel(ctxtimeout) + defer cancel() + + // next we opens a connection to the server to watch for + // cancellation requests. If a build is cancelled the running + // stage should also be cancelled. + go func() { + done, _ := s.Client.Watch(ctxdone, data.Build.ID) + if done { + cancel() + log.Debugln("received cancellation") + } else { + log.Debugln("done listening for cancellations") + } + }() + + envs := environ.Combine( + s.Environ, + environ.System(data.System), + environ.Repo(data.Repo), + environ.Build(data.Build), + environ.Stage(stage), + environ.Link(data.Repo, data.Build, data.System), + data.Build.Params, + ) + + // string substitution function ensures that string + // replacement variables are escaped and quoted if they + // contain a newline character. + subf := func(k string) string { + v := envs[k] + if strings.Contains(v, "\n") { + v = fmt.Sprintf("%q", v) + } + return v + } + + state := &pipeline.State{ + Build: data.Build, + Stage: stage, + Repo: data.Repo, + System: data.System, + } + + // evaluates whether or not the agent can process the + // pipeline. An agent may choose to reject a repository + // or build for security reasons. + if s.Match != nil && s.Match(data.Repo, data.Build) == false { + log.Error("cannot process stage, access denied") + state.FailAll(errors.New("insufficient permission to run the pipeline")) + return s.Reporter.ReportStage(noContext, state) + } + + // evaluates string replacement expressions and returns an + // update configuration file string. + config, err := envsubst.Eval(string(data.Config.Data), subf) + if err != nil { + log.WithError(err).Error("cannot emulate bash substitution") + state.FailAll(err) + return s.Reporter.ReportStage(noContext, state) + } + + // parse the yaml configuration file. + manifest, err := manifest.ParseString(config) + if err != nil { + log.WithError(err).Error("cannot parse configuration file") + state.FailAll(err) + return s.Reporter.ReportStage(noContext, state) + } + + // find the named stage in the yaml configuration file. + resource, err := resource.Lookup(stage.Name, manifest) + if err != nil { + log.WithError(err).Error("cannot find pipeline resource") + state.FailAll(err) + return s.Reporter.ReportStage(noContext, state) + } + + secrets := secret.Combine( + secret.Static(data.Secrets), + secret.Encrypted(), + s.Secret, + ) + + // compile the yaml configuration file to an intermediate + // representation, and then + comp := &compiler.Compiler{ + Pipeline: resource, + Manifest: manifest, + Environ: s.Environ, + Build: data.Build, + Stage: stage, + Repo: data.Repo, + System: data.System, + Netrc: data.Netrc, + Secret: secrets, + } + + spec := comp.Compile(ctx) + for _, src := range spec.Steps { + // steps that are skipped are ignored and are not stored + // in the drone database, nor displayed in the UI. + if src.RunPolicy == engine.RunNever { + continue + } + stage.Steps = append(stage.Steps, &drone.Step{ + Name: src.Name, + Number: len(stage.Steps) + 1, + StageID: stage.ID, + Status: drone.StatusPending, + ErrIgnore: src.IgnoreErr, + }) + } + + stage.Started = time.Now().Unix() + stage.Status = drone.StatusRunning + if err := s.Client.Update(ctx, stage); err != nil { + log.WithError(err).Error("cannot update stage") + return err + } + + log.Debug("updated stage to running") + + ctxcancel = logger.WithContext(ctxcancel, log) + err = s.Execer.Exec(ctxcancel, spec, state) + if err != nil { + log.WithError(err).Debug("stage failed") + return err + } + log.Debug("updated stage to complete") + return nil +} diff --git a/runtime/runner_test.go b/runtime/runner_test.go new file mode 100644 index 0000000..69aede2 --- /dev/null +++ b/runtime/runner_test.go @@ -0,0 +1,7 @@ +// Code generated automatically. DO NOT EDIT. + +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Polyform License +// that can be found in the LICENSE file. + +package runtime diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..c5d08eb --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# disable go modules +export GOPATH="" + +# disable cgo +export CGO_ENABLED=0 + +set -e +set -x + +# linux +GOOS=linux GOARCH=amd64 go build -o release/linux/amd64/drone-runner-docker +GOOS=linux GOARCH=arm64 go build -o release/linux/arm64/drone-runner-docker +GOOS=linux GOARCH=arm go build -o release/linux/arm/drone-runner-docker + +# windows +GOOS=windows go build -o release/windows/amd64/drone-runner-docker.exe