initial commit [ci skip]

This commit is contained in:
Brad Rydzewski
2019-10-10 19:01:58 -07:00
parent 56c135e4ae
commit 43bbf6e78c
95 changed files with 6579 additions and 1 deletions

32
command/command.go Normal file
View File

@@ -0,0 +1,32 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package command
import (
"context"
"os"
"github.com/drone-runners/drone-runner-docker/command/daemon"
"gopkg.in/alecthomas/kingpin.v2"
)
// program version
var version = "0.0.0"
// empty context
var nocontext = context.Background()
// Command parses the command line arguments and then executes a
// subcommand program.
func Command() {
app := kingpin.New("drone", "drone exec runner")
registerCompile(app)
registerExec(app)
daemon.Register(app)
kingpin.Version(version)
kingpin.MustParse(app.Parse(os.Args[1:]))
}

122
command/compile.go Normal file
View File

@@ -0,0 +1,122 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package command
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"github.com/drone-runners/drone-runner-docker/command/internal"
"github.com/drone-runners/drone-runner-docker/engine/compiler"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone/envsubst"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/secret"
"gopkg.in/alecthomas/kingpin.v2"
)
type compileCommand struct {
*internal.Flags
Source *os.File
Environ map[string]string
Secrets map[string]string
}
func (c *compileCommand) run(*kingpin.ParseContext) error {
rawsource, err := ioutil.ReadAll(c.Source)
if err != nil {
return err
}
envs := environ.Combine(
c.Environ,
environ.System(c.System),
environ.Repo(c.Repo),
environ.Build(c.Build),
environ.Stage(c.Stage),
environ.Link(c.Repo, c.Build, c.System),
c.Build.Params,
)
// string substitution function ensures that string
// replacement variables are escaped and quoted if they
// contain newlines.
subf := func(k string) string {
v := envs[k]
if strings.Contains(v, "\n") {
v = fmt.Sprintf("%q", v)
}
return v
}
// evaluates string replacement expressions and returns an
// update configuration.
config, err := envsubst.Eval(string(rawsource), subf)
if err != nil {
return err
}
// parse and lint the configuration
manifest, err := manifest.ParseString(config)
if err != nil {
return err
}
// a configuration can contain multiple pipelines.
// get a specific pipeline resource for execution.
resource, err := resource.Lookup(c.Stage.Name, manifest)
if err != nil {
return err
}
// compile the pipeline to an intermediate representation.
comp := &compiler.Compiler{
Pipeline: resource,
Manifest: manifest,
Build: c.Build,
Netrc: c.Netrc,
Repo: c.Repo,
Stage: c.Stage,
System: c.System,
Environ: c.Environ,
Secret: secret.StaticVars(c.Secrets),
}
spec := comp.Compile(nocontext)
// encode the pipeline in json format and print to the
// console for inspection.
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(spec)
return nil
}
func registerCompile(app *kingpin.Application) {
c := new(compileCommand)
c.Environ = map[string]string{}
c.Secrets = map[string]string{}
cmd := app.Command("compile", "compile the yaml file").
Action(c.run)
cmd.Flag("source", "source file location").
Default(".drone.yml").
FileVar(&c.Source)
cmd.Flag("secrets", "secret parameters").
StringMapVar(&c.Secrets)
cmd.Flag("environ", "environment variables").
StringMapVar(&c.Environ)
// shared pipeline flags
c.Flags = internal.ParseFlags(cmd)
}

94
command/daemon/config.go Normal file
View File

@@ -0,0 +1,94 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package daemon
import (
"fmt"
"os"
"github.com/kelseyhightower/envconfig"
)
// Config stores the system configuration.
type Config struct {
Debug bool `envconfig:"DRONE_DEBUG"`
Trace bool `envconfig:"DRONE_TRACE"`
Logger struct {
File string `envconfig:"DRONE_LOG_FILE"`
MaxAge int `envconfig:"DRONE_LOG_FILE_MAX_AGE" default:"1"`
MaxBackups int `envconfig:"DRONE_LOG_FILE_MAX_BACKUPS" default:"1"`
MaxSize int `envconfig:"DRONE_LOG_FILE_MAX_SIZE" default:"100"`
}
Client struct {
Address string `ignored:"true"`
Proto string `envconfig:"DRONE_RPC_PROTO" default:"http"`
Host string `envconfig:"DRONE_RPC_HOST" required:"true"`
Secret string `envconfig:"DRONE_RPC_SECRET" required:"true"`
SkipVerify bool `envconfig:"DRONE_RPC_SKIP_VERIFY"`
Dump bool `envconfig:"DRONE_RPC_DUMP_HTTP"`
DumpBody bool `envconfig:"DRONE_RPC_DUMP_HTTP_BODY"`
}
Dashboard struct {
Disabled bool `envconfig:"DRONE_UI_DISABLE"`
Username string `envconfig:"DRONE_UI_USERNAME"`
Password string `envconfig:"DRONE_UI_PASSWORD"`
Realm string `envconfig:"DRONE_UI_REALM" default:"MyRealm"`
}
Server struct {
Proto string `envconfig:"DRONE_SERVER_PROTO"`
Host string `envconfig:"DRONE_SERVER_HOST"`
Port string `envconfig:"DRONE_SERVER_PORT" default:":3000"`
Acme bool `envconfig:"DRONE_SERVER_ACME"`
}
Keypair struct {
Public string `envconfig:"DRONE_PUBLIC_KEY_FILE"`
Private string `envconfig:"DRONE_PRIVATE_KEY_FILE"`
}
Runner struct {
Name string `envconfig:"DRONE_RUNNER_NAME"`
Capacity int `envconfig:"DRONE_RUNNER_CAPACITY" default:"10"`
Procs int64 `envconfig:"DRONE_RUNNER_MAX_PROCS"`
Labels map[string]string `envconfig:"DRONE_RUNNER_LABELS"`
Environ map[string]string `envconfig:"DRONE_RUNNER_ENVIRON"`
}
Limit struct {
Repos []string `envconfig:"DRONE_LIMIT_REPOS"`
Events []string `envconfig:"DRONE_LIMIT_EVENTS"`
Trusted bool `envconfig:"DRONE_LIMIT_TRUSTED"`
}
Secret struct {
Endpoint string `envconfig:"DRONE_SECRET_PLUGIN_ENDPOINT"`
Token string `envconfig:"DRONE_SECRET_PLUGIN_TOKEN"`
SkipVerify bool `envconfig:"DRONE_SECRET_PLUGIN_SKIP_VERIFY"`
}
}
func fromEnviron() (Config, error) {
var config Config
err := envconfig.Process("", &config)
if err != nil {
return config, err
}
if config.Runner.Name == "" {
config.Runner.Name, _ = os.Hostname()
}
if config.Dashboard.Password == "" {
config.Dashboard.Disabled = true
}
config.Client.Address = fmt.Sprintf(
"%s://%s",
config.Client.Proto,
config.Client.Host,
)
return config, nil
}

204
command/daemon/daemon.go Normal file
View File

@@ -0,0 +1,204 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package daemon
import (
"context"
"time"
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/internal/match"
"github.com/drone-runners/drone-runner-docker/runtime"
"github.com/drone/runner-go/client"
"github.com/drone/runner-go/handler/router"
"github.com/drone/runner-go/logger"
loghistory "github.com/drone/runner-go/logger/history"
"github.com/drone/runner-go/pipeline/history"
"github.com/drone/runner-go/pipeline/remote"
"github.com/drone/runner-go/secret"
"github.com/drone/runner-go/server"
"github.com/drone/signal"
"github.com/joho/godotenv"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"gopkg.in/alecthomas/kingpin.v2"
)
// empty context.
var nocontext = context.Background()
type daemonCommand struct {
envfile string
}
func (c *daemonCommand) run(*kingpin.ParseContext) error {
// load environment variables from file.
godotenv.Load(c.envfile)
// load the configuration from the environment
config, err := fromEnviron()
if err != nil {
return err
}
// setup the global logrus logger.
setupLogger(config)
cli := client.New(
config.Client.Address,
config.Client.Secret,
config.Client.SkipVerify,
)
if config.Client.Dump {
cli.Dumper = logger.StandardDumper(
config.Client.DumpBody,
)
}
cli.Logger = logger.Logrus(
logrus.NewEntry(
logrus.StandardLogger(),
),
)
engine, err := engine.New(config.Keypair.Public, config.Keypair.Private)
if err != nil {
return err
}
remote := remote.New(cli)
tracer := history.New(remote)
hook := loghistory.New()
logrus.AddHook(hook)
poller := &runtime.Poller{
Client: cli,
Runner: &runtime.Runner{
Client: cli,
Environ: config.Runner.Environ,
Machine: config.Runner.Name,
Reporter: tracer,
Match: match.Func(
config.Limit.Repos,
config.Limit.Events,
config.Limit.Trusted,
),
Secret: secret.External(
config.Secret.Endpoint,
config.Secret.Token,
config.Secret.SkipVerify,
),
Execer: runtime.NewExecer(
tracer,
remote,
engine,
config.Runner.Procs,
),
},
Filter: &client.Filter{
Kind: resource.Kind,
Type: resource.Type,
Labels: config.Runner.Labels,
},
}
ctx, cancel := context.WithCancel(nocontext)
defer cancel()
// listen for termination signals to gracefully shutdown
// the runner daemon.
ctx = signal.WithContextFunc(ctx, func() {
println("received signal, terminating process")
cancel()
})
var g errgroup.Group
server := server.Server{
Addr: config.Server.Port,
Handler: router.New(tracer, hook, router.Config{
Username: config.Dashboard.Username,
Password: config.Dashboard.Password,
Realm: config.Dashboard.Realm,
}),
}
logrus.WithField("addr", config.Server.Port).
Infoln("starting the server")
g.Go(func() error {
return server.ListenAndServe(ctx)
})
// Ping the server and block until a successful connection
// to the server has been established.
for {
err := cli.Ping(ctx, config.Runner.Name)
select {
case <-ctx.Done():
return nil
default:
}
if ctx.Err() != nil {
break
}
if err != nil {
logrus.WithError(err).
Errorln("cannot ping the remote server")
time.Sleep(time.Second)
} else {
logrus.Infoln("successfully pinged the remote server")
break
}
}
g.Go(func() error {
logrus.WithField("capacity", config.Runner.Capacity).
WithField("endpoint", config.Client.Address).
WithField("kind", resource.Kind).
WithField("type", resource.Type).
Infoln("polling the remote server")
poller.Poll(ctx, config.Runner.Capacity)
return nil
})
err = g.Wait()
if err != nil {
logrus.WithError(err).
Errorln("shutting down the server")
}
return err
}
// helper function configures the global logger from
// the loaded configuration.
func setupLogger(config Config) {
logger.Default = logger.Logrus(
logrus.NewEntry(
logrus.StandardLogger(),
),
)
if config.Debug {
logrus.SetLevel(logrus.DebugLevel)
}
if config.Trace {
logrus.SetLevel(logrus.TraceLevel)
}
}
// Register the daemon command.
func Register(app *kingpin.Application) {
c := new(daemonCommand)
cmd := app.Command("daemon", "starts the runner daemon").
Default().
Action(c.run)
cmd.Arg("envfile", "load the environment variable file").
Default("").
StringVar(&c.envfile)
}

234
command/exec.go Normal file
View File

@@ -0,0 +1,234 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package command
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"time"
"github.com/drone-runners/drone-runner-docker/command/internal"
"github.com/drone-runners/drone-runner-docker/engine"
"github.com/drone-runners/drone-runner-docker/engine/compiler"
"github.com/drone-runners/drone-runner-docker/engine/resource"
"github.com/drone-runners/drone-runner-docker/runtime"
"github.com/drone/drone-go/drone"
"github.com/drone/envsubst"
"github.com/drone/runner-go/environ"
"github.com/drone/runner-go/logger"
"github.com/drone/runner-go/manifest"
"github.com/drone/runner-go/pipeline"
"github.com/drone/runner-go/pipeline/console"
"github.com/drone/runner-go/secret"
"github.com/drone/signal"
"github.com/mattn/go-isatty"
"github.com/sirupsen/logrus"
"gopkg.in/alecthomas/kingpin.v2"
)
type execCommand struct {
*internal.Flags
Source *os.File
Environ map[string]string
Secrets map[string]string
Pretty bool
Procs int64
Debug bool
Trace bool
Dump bool
PublicKey string
PrivateKey string
}
func (c *execCommand) run(*kingpin.ParseContext) error {
rawsource, err := ioutil.ReadAll(c.Source)
if err != nil {
return err
}
envs := environ.Combine(
c.Environ,
environ.System(c.System),
environ.Repo(c.Repo),
environ.Build(c.Build),
environ.Stage(c.Stage),
environ.Link(c.Repo, c.Build, c.System),
c.Build.Params,
)
// string substitution function ensures that string
// replacement variables are escaped and quoted if they
// contain newlines.
subf := func(k string) string {
v := envs[k]
if strings.Contains(v, "\n") {
v = fmt.Sprintf("%q", v)
}
return v
}
// evaluates string replacement expressions and returns an
// update configuration.
config, err := envsubst.Eval(string(rawsource), subf)
if err != nil {
return err
}
// parse and lint the configuration.
manifest, err := manifest.ParseString(config)
if err != nil {
return err
}
// a configuration can contain multiple pipelines.
// get a specific pipeline resource for execution.
resource, err := resource.Lookup(c.Stage.Name, manifest)
if err != nil {
return err
}
// compile the pipeline to an intermediate representation.
comp := &compiler.Compiler{
Pipeline: resource,
Manifest: manifest,
Build: c.Build,
Netrc: c.Netrc,
Repo: c.Repo,
Stage: c.Stage,
System: c.System,
Environ: c.Environ,
Secret: secret.StaticVars(c.Secrets),
}
spec := comp.Compile(nocontext)
// create a step object for each pipeline step.
for _, step := range spec.Steps {
if step.RunPolicy == engine.RunNever {
continue
}
c.Stage.Steps = append(c.Stage.Steps, &drone.Step{
StageID: c.Stage.ID,
Number: len(c.Stage.Steps) + 1,
Name: step.Name,
Status: drone.StatusPending,
ErrIgnore: step.IgnoreErr,
})
}
// configures the pipeline timeout.
timeout := time.Duration(c.Repo.Timeout) * time.Minute
ctx, cancel := context.WithTimeout(nocontext, timeout)
defer cancel()
// listen for operating system signals and cancel execution
// when received.
ctx = signal.WithContextFunc(ctx, func() {
println("received signal, terminating process")
cancel()
})
state := &pipeline.State{
Build: c.Build,
Stage: c.Stage,
Repo: c.Repo,
System: c.System,
}
// enable debug logging
logrus.SetLevel(logrus.WarnLevel)
if c.Debug {
logrus.SetLevel(logrus.DebugLevel)
}
if c.Trace {
logrus.SetLevel(logrus.TraceLevel)
}
logger.Default = logger.Logrus(
logrus.NewEntry(
logrus.StandardLogger(),
),
)
engine, err := engine.New(c.PublicKey, c.PrivateKey)
if err != nil {
return err
}
err = runtime.NewExecer(
pipeline.NopReporter(),
console.New(c.Pretty),
engine,
c.Procs,
).Exec(ctx, spec, state)
if c.Dump {
dump(state)
}
if err != nil {
return err
}
switch state.Stage.Status {
case drone.StatusError, drone.StatusFailing, drone.StatusKilled:
os.Exit(1)
}
return nil
}
func dump(v interface{}) {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(v)
}
func registerExec(app *kingpin.Application) {
c := new(execCommand)
c.Environ = map[string]string{}
c.Secrets = map[string]string{}
cmd := app.Command("exec", "executes a pipeline").
Action(c.run)
cmd.Arg("source", "source file location").
Default(".drone.yml").
FileVar(&c.Source)
cmd.Flag("secrets", "secret parameters").
StringMapVar(&c.Secrets)
cmd.Flag("environ", "environment variables").
StringMapVar(&c.Environ)
cmd.Flag("public-key", "public key file path").
ExistingFileVar(&c.PublicKey)
cmd.Flag("private-key", "private key file path").
ExistingFileVar(&c.PrivateKey)
cmd.Flag("debug", "enable debug logging").
BoolVar(&c.Debug)
cmd.Flag("trace", "enable trace logging").
BoolVar(&c.Trace)
cmd.Flag("dump", "dump the pipeline state to stdout").
BoolVar(&c.Dump)
cmd.Flag("pretty", "pretty print the output").
Default(
fmt.Sprint(
isatty.IsTerminal(
os.Stdout.Fd(),
),
),
).BoolVar(&c.Pretty)
// shared pipeline flags
c.Flags = internal.ParseFlags(cmd)
}

103
command/internal/flags.go Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2019 Drone.IO Inc. All rights reserved.
// Use of this source code is governed by the Polyform License
// that can be found in the LICENSE file.
package internal
import (
"fmt"
"time"
"github.com/drone/drone-go/drone"
"gopkg.in/alecthomas/kingpin.v2"
)
// Flags maps
type Flags struct {
Build *drone.Build
Netrc *drone.Netrc
Repo *drone.Repo
Stage *drone.Stage
System *drone.System
}
// ParseFlags parses the flags from the command args.
func ParseFlags(cmd *kingpin.CmdClause) *Flags {
f := &Flags{
Build: &drone.Build{},
Netrc: &drone.Netrc{},
Repo: &drone.Repo{},
Stage: &drone.Stage{},
System: &drone.System{},
}
now := fmt.Sprint(
time.Now().Unix(),
)
cmd.Flag("repo-id", "repo id").Default("1").Int64Var(&f.Repo.ID)
cmd.Flag("repo-namespace", "repo namespace").Default("").StringVar(&f.Repo.Namespace)
cmd.Flag("repo-name", "repo name").Default("").StringVar(&f.Repo.Name)
cmd.Flag("repo-slug", "repo slug").Default("").StringVar(&f.Repo.Slug)
cmd.Flag("repo-http", "repo http clone url").Default("").StringVar(&f.Repo.HTTPURL)
cmd.Flag("repo-ssh", "repo ssh clone url").Default("").StringVar(&f.Repo.SSHURL)
cmd.Flag("repo-link", "repo link").Default("").StringVar(&f.Repo.Link)
cmd.Flag("repo-branch", "repo branch").Default("").StringVar(&f.Repo.Branch)
cmd.Flag("repo-private", "repo private").Default("false").BoolVar(&f.Repo.Private)
cmd.Flag("repo-visibility", "repo visibility").Default("").StringVar(&f.Repo.Visibility)
cmd.Flag("repo-trusted", "repo trusted").Default("false").BoolVar(&f.Repo.Trusted)
cmd.Flag("repo-protected", "repo protected").Default("false").BoolVar(&f.Repo.Protected)
cmd.Flag("repo-timeout", "repo timeout in minutes").Default("60").Int64Var(&f.Repo.Timeout)
cmd.Flag("repo-created", "repo created").Default(now).Int64Var(&f.Repo.Created)
cmd.Flag("repo-updated", "repo updated").Default(now).Int64Var(&f.Repo.Updated)
cmd.Flag("build-id", "build id").Default("1").Int64Var(&f.Build.ID)
cmd.Flag("build-number", "build number").Default("1").Int64Var(&f.Build.Number)
cmd.Flag("build-parent", "build parent").Default("0").Int64Var(&f.Build.Parent)
cmd.Flag("build-event", "build event").Default("push").StringVar(&f.Build.Event)
cmd.Flag("build-action", "build action").Default("").StringVar(&f.Build.Action)
cmd.Flag("build-cron", "build cron trigger").Default("").StringVar(&f.Build.Cron)
cmd.Flag("build-target", "build deploy target").Default("").StringVar(&f.Build.Deploy)
cmd.Flag("build-created", "build created").Default(now).Int64Var(&f.Build.Created)
cmd.Flag("build-updated", "build updated").Default(now).Int64Var(&f.Build.Updated)
cmd.Flag("commit-sender", "commit sender").Default("").StringVar(&f.Build.Sender)
cmd.Flag("commit-link", "commit link").Default("").StringVar(&f.Build.Link)
cmd.Flag("commit-title", "commit title").Default("").StringVar(&f.Build.Title)
cmd.Flag("commit-message", "commit message").Default("").StringVar(&f.Build.Message)
cmd.Flag("commit-before", "commit before").Default("").StringVar(&f.Build.Before)
cmd.Flag("commit-after", "commit after").Default("").StringVar(&f.Build.After)
cmd.Flag("commit-ref", "commit ref").Default("").StringVar(&f.Build.Ref)
cmd.Flag("commit-fork", "commit fork").Default("").StringVar(&f.Build.Fork)
cmd.Flag("commit-source", "commit source branch").Default("").StringVar(&f.Build.Source)
cmd.Flag("commit-target", "commit target branch").Default("").StringVar(&f.Build.Target)
cmd.Flag("author-login", "commit author login").Default("").StringVar(&f.Build.Author)
cmd.Flag("author-name", "commit author name").Default("").StringVar(&f.Build.AuthorName)
cmd.Flag("author-email", "commit author email").Default("").StringVar(&f.Build.AuthorEmail)
cmd.Flag("author-avatar", "commit author avatar").Default("").StringVar(&f.Build.AuthorAvatar)
cmd.Flag("stage-id", "stage id").Default("1").Int64Var(&f.Stage.ID)
cmd.Flag("stage-number", "stage number").Default("1").IntVar(&f.Stage.Number)
cmd.Flag("stage-kind", "stage kind").Default("").StringVar(&f.Stage.Kind)
cmd.Flag("stage-type", "stage type").Default("").StringVar(&f.Stage.Type)
cmd.Flag("stage-name", "stage name").Default("default").StringVar(&f.Stage.Name)
cmd.Flag("stage-os", "stage os").Default("").StringVar(&f.Stage.OS)
cmd.Flag("stage-arch", "stage arch").Default("").StringVar(&f.Stage.Arch)
cmd.Flag("stage-variant", "stage variant").Default("").StringVar(&f.Stage.Variant)
cmd.Flag("stage-kernel", "stage kernel").Default("").StringVar(&f.Stage.Kernel)
cmd.Flag("stage-created", "stage created").Default(now).Int64Var(&f.Stage.Created)
cmd.Flag("stage-updated", "stage updated").Default(now).Int64Var(&f.Stage.Updated)
cmd.Flag("netrc-username", "netrc username").Default("").StringVar(&f.Netrc.Login)
cmd.Flag("netrc-password", "netrc password").Default("").StringVar(&f.Netrc.Password)
cmd.Flag("netrc-machine", "netrc machine").Default("").StringVar(&f.Netrc.Machine)
cmd.Flag("system-host", "server host").Default("").StringVar(&f.System.Host)
cmd.Flag("system-proto", "server proto").Default("").StringVar(&f.System.Proto)
cmd.Flag("system-link", "server link").Default("").StringVar(&f.System.Link)
cmd.Flag("system-version", "server version").Default("").StringVar(&f.System.Version)
return f
}