From 28e305d9f52b8687f17201b1341ba2427915ea80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Wed, 7 Nov 2018 22:23:14 +0100 Subject: [PATCH 1/5] Rearrange build steps --- Dockerfile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1136cf6..8710de1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,20 +17,6 @@ ENV SALT_CONFS_DIR="${SALT_DATA_DIR}/config" \ SALT_KEYS_DIR="${SALT_DATA_DIR}/keys" \ SALT_BASE_DIR="${SALT_DATA_DIR}/srv" -# Bootstrap script options: -# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options -## -M: install Salt Master by default -## -N: Do not install salt-minion -## -X: Do not start daemons after installation -## -U: Fully upgrade the system prior to bootstrapping Salt -## -V: Install Salt into virtualenv -## -a: Pip install all Python pkg dependencies for Salt -ENV SALT_BOOTSTRAP_OPTS='-M -N -X -U -Va' - -# Release version to install -# https://github.com/saltstack/salt/releases -ENV SALT_GIT_RELEASE="v2018.3.3" - # Set non interactive mode ENV DEBIAN_FRONTEND=noninteractive @@ -49,6 +35,20 @@ RUN update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \ RUN mkdir -p ${SALT_BUILD_DIR} WORKDIR ${SALT_BUILD_DIR} +# Bootstrap script options: +# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options +## -M: install Salt Master by default +## -N: Do not install salt-minion +## -X: Do not start daemons after installation +## -U: Fully upgrade the system prior to bootstrapping Salt +## -V: Install Salt into virtualenv +## -a: Pip install all Python pkg dependencies for Salt +ENV SALT_BOOTSTRAP_OPTS='-M -N -X -U -Va' + +# Release version to install +# https://github.com/saltstack/salt/releases +ENV SALT_GIT_RELEASE="v2018.3.3" + RUN curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com RUN sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} git ${SALT_GIT_RELEASE} From 45df5e99a515777b9e6f2c050ed66edb5619f98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Wed, 7 Nov 2018 22:25:29 +0100 Subject: [PATCH 2/5] Change saltstack bootstrap options --- Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8710de1..ba7716c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,17 +40,16 @@ WORKDIR ${SALT_BUILD_DIR} ## -M: install Salt Master by default ## -N: Do not install salt-minion ## -X: Do not start daemons after installation -## -U: Fully upgrade the system prior to bootstrapping Salt -## -V: Install Salt into virtualenv -## -a: Pip install all Python pkg dependencies for Salt -ENV SALT_BOOTSTRAP_OPTS='-M -N -X -U -Va' +## -P: Allow pip based installations +## -x: Changes the python version used to install a git version of salt +ENV SALT_BOOTSTRAP_OPTS='-M -N -X -P -x python3' # Release version to install # https://github.com/saltstack/salt/releases -ENV SALT_GIT_RELEASE="v2018.3.3" +ENV SALT_VERSION="2018.3.3" RUN curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com -RUN sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} git ${SALT_GIT_RELEASE} +RUN sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} stable ${SALT_VERSION} # Salt user RUN useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER} From 1bb94cac4ea2dcb8d11e1260e5d0b9576a2eaeda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Wed, 7 Nov 2018 22:34:21 +0100 Subject: [PATCH 3/5] Move build steps into assets/build/install.sh --- Dockerfile | 34 +++++++++++----------------------- assets/build/install.sh | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 23 deletions(-) create mode 100755 assets/build/install.sh diff --git a/Dockerfile b/Dockerfile index ba7716c..dcfdc5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ LABEL maintainer="carlos.alvaro@citelan.es" LABEL description="SaltStack master" LABEL version="2018.3.3" +# https://github.com/saltstack/salt/releases +ENV SALT_VERSION="2018.3.3" + ENV SALT_DOCKER_DIR="/etc/docker-salt" \ SALT_ROOT_DIR="/etc/salt" \ SALT_USER="salt" \ @@ -20,10 +23,13 @@ ENV SALT_CONFS_DIR="${SALT_DATA_DIR}/config" \ # Set non interactive mode ENV DEBIAN_FRONTEND=noninteractive +RUN mkdir -p ${SALT_BUILD_DIR} +WORKDIR ${SALT_BUILD_DIR} + # Install packages RUN apt-get update RUN apt-get install --yes --quiet --no-install-recommends \ - ca-certificates apt-transport-https curl git vim locales \ + ca-certificates wget apt-transport-https git locales \ openssh-client python3 python-git # Configure locales @@ -32,29 +38,10 @@ RUN update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \ dpkg-reconfigure locales # Install saltstack -RUN mkdir -p ${SALT_BUILD_DIR} -WORKDIR ${SALT_BUILD_DIR} - -# Bootstrap script options: -# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options -## -M: install Salt Master by default -## -N: Do not install salt-minion -## -X: Do not start daemons after installation -## -P: Allow pip based installations -## -x: Changes the python version used to install a git version of salt -ENV SALT_BOOTSTRAP_OPTS='-M -N -X -P -x python3' - -# Release version to install -# https://github.com/saltstack/salt/releases -ENV SALT_VERSION="2018.3.3" - -RUN curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com -RUN sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} stable ${SALT_VERSION} - -# Salt user -RUN useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER} -RUN chown -R ${SALT_USER}: ${SALT_ROOT_DIR} +COPY assets/build ${SALT_BUILD_DIR} +RUN bash ${SALT_BUILD_DIR}/install.sh +# Shared resources EXPOSE 4505/tcp 4506/tcp RUN mkdir -p ${SALT_DATA_DIR} ${SALT_BASE_DIR} ${SALT_KEYS_DIR} ${SALT_CONFS_DIR} VOLUME [ "${SALT_BASE_DIR}" "${SALT_KEYS_DIR}" "${SALT_CONFS_DIR}" ] @@ -65,6 +52,7 @@ RUN chmod -R +x ${SALT_RUNTIME_DIR} # Cleaning tasks RUN apt-get clean --yes RUN rm -rf /var/lib/apt/lists/* +RUN rm -rf ${SALT_BUILD_DIR}/* # Entrypoint COPY entrypoint.sh /sbin/entrypoint.sh diff --git a/assets/build/install.sh b/assets/build/install.sh new file mode 100755 index 0000000..17b529e --- /dev/null +++ b/assets/build/install.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +# Salt user +echo "Creating ${SALT_USER} user ..." +useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER} + +# Bootstrap script options: +# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options +## -M: install Salt Master by default +## -N: Do not install salt-minion +## -X: Do not start daemons after installation +## -P: Allow pip based installations +## -x: Changes the python version used to install a git version of salt +SALT_BOOTSTRAP_OPTS='-M -N -X -P -x python3' + +echo "Installing saltstack ..." +wget -O bootstrap-salt.sh https://bootstrap.saltstack.com +sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} stable ${SALT_VERSION} +chown -R ${SALT_USER}: ${SALT_ROOT_DIR} From 2f701fbef86ac81a3ee296edb03a97c5e1270118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Wed, 7 Nov 2018 22:42:56 +0100 Subject: [PATCH 4/5] Add support for pygit2 --- Dockerfile | 11 ++++++++--- assets/build/install.sh | 30 +++++++++++++++++++++++++++++- assets/runtime/functions.sh | 3 ++- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index dcfdc5c..662fa5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,11 @@ LABEL description="SaltStack master" LABEL version="2018.3.3" # https://github.com/saltstack/salt/releases -ENV SALT_VERSION="2018.3.3" +ENV SALT_VERSION="2018.3.3" \ + PYTHON_VERSION="3.5" \ + LIBSSH2_VERSION="1.8.0" \ + LIBGIT2_VERSION="0.27.7" \ + PYGIT2_VERSION="0.27.2" ENV SALT_DOCKER_DIR="/etc/docker-salt" \ SALT_ROOT_DIR="/etc/salt" \ @@ -29,8 +33,9 @@ WORKDIR ${SALT_BUILD_DIR} # Install packages RUN apt-get update RUN apt-get install --yes --quiet --no-install-recommends \ - ca-certificates wget apt-transport-https git locales \ - openssh-client python3 python-git + ca-certificates wget locales pkg-config openssh-client \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-dev \ + python3-pip python3-setuptools python3-wheel # Configure locales RUN update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \ diff --git a/assets/build/install.sh b/assets/build/install.sh index 17b529e..cb1de6b 100755 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -2,6 +2,34 @@ set -e +# Install build dependencies +echo "Installing dependencies ..." +BUILD_DEPENDENCIES="gnupg git cmake gcc g++ make \ + libhttp-parser-dev libssl-dev zlib1g-dev \ + libzmq-dev libcurl4-openssl-dev libffi-dev" + +apt-get install --yes --quiet --no-install-recommends ${BUILD_DEPENDENCIES} + +# Compile libssh2 +echo "Building libssh2 v${LIBSSH2_VERSION} ..." +wget https://github.com/libssh2/libssh2/archive/libssh2-${LIBSSH2_VERSION}.tar.gz +tar xzf libssh2-${LIBSSH2_VERSION}.tar.gz +cd libssh2-libssh2-${LIBSSH2_VERSION}/ +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DENABLE_ZLIB_COMPRESSION=ON . +cmake --build . --target install + +# Compile libgit2 +echo "Building libgit2 v${LIBGIT2_VERSION} ..." +wget https://github.com/libgit2/libgit2/archive/v${LIBGIT2_VERSION}.tar.gz +tar xzf v${LIBGIT2_VERSION}.tar.gz +cd libgit2-${LIBGIT2_VERSION}/ +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DBUILD_CLAR=OFF -DTHREADSAFE=ON . +cmake --build . --target install + +# Install python packages +echo "Installing python packages ..." +pip3 install "pygit2==v${PYGIT2_VERSION}" gitpython + # Salt user echo "Creating ${SALT_USER} user ..." useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER} @@ -13,7 +41,7 @@ useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER} ## -X: Do not start daemons after installation ## -P: Allow pip based installations ## -x: Changes the python version used to install a git version of salt -SALT_BOOTSTRAP_OPTS='-M -N -X -P -x python3' +SALT_BOOTSTRAP_OPTS="-M -N -X -P -x python${PYTHON_VERSION}" echo "Installing saltstack ..." wget -O bootstrap-salt.sh https://bootstrap.saltstack.com diff --git a/assets/runtime/functions.sh b/assets/runtime/functions.sh index 796cb8d..3085b87 100755 --- a/assets/runtime/functions.sh +++ b/assets/runtime/functions.sh @@ -64,7 +64,7 @@ EOF salt-key --gen-signature --auto-create --pub ${SALT_KEYS_DIR}/master.pub --signature-path ${SALT_KEYS_DIR} fi - for pub_key in $(find ${SALT_KEYS_DIR} -type f -maxdepth 2); do + for pub_key in $(find ${SALT_KEYS_DIR} -type f -maxdepth 1); do if [[ ${pub_key} =~ .*\.pem$ ]]; then chmod 400 ${pub_key} else @@ -72,6 +72,7 @@ EOF fi done + find ${SALT_KEYS_DIR}/minions* -type f -maxdepth 1 -exec chmod 644 {} \; find ${SALT_HOME} -path ${SALT_KEYS_DIR}/\* -prune -o -print0 | xargs -0 chown -h ${SALT_USER}: } From 5ffa4393214fa7e2ef3e192ec368d54f4b01dd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Wed, 7 Nov 2018 22:43:19 +0100 Subject: [PATCH 5/5] Update README.md --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2f922ef..770df63 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ For other methods to install SaltStack please refer to the [Official SaltStack I - [Master Signed Keys](#master-signed-keys) - [Host Mapping](#host-mapping) - [Git Fileserver](#git-fileserver) + - [GitPython](#gitpython) + - [PyGit2](#pygit2) - [Available Configuration Parameters](#available-configuration-parameters) - [Usage](#usage) - [Shell Access](#shell-access) @@ -131,14 +133,44 @@ docker run --name salt_stack -it --rm \ ### Git Fileserver -This image uses [GitPython](https://github.com/gitpython-developers/GitPython) as gitfs backend to allow Salt to serve files from git repositories. +This image uses [GitPython](https://github.com/gitpython-developers/GitPython) and [PyGit2](https://www.pygit2.org) as gitfs backends to allow Salt to serve files from git repositories. It can be enabled by adding `gitfs` to the [`fileserver_backend`](https://docs.saltstack.com/en/latest/ref/configuration/master.html#std:conf_master-fileserver_backend) list (see [Available Configuration Parameters](#available-configuration-parameters)), and configuring one or more repositories in [`gitfs_remotes`](https://docs.saltstack.com/en/latest/ref/configuration/master.html#std:conf_master-gitfs_remotes). -As the backend for gitfs is GitPython, then an ssh key is needed. The default name for the ssh key is `gitfs_ssh` but it can be changed with the env variables `SALT_GITFS_SSH_PRIVATE_KEY` and `SALT_GITFS_SSH_PUBLIC_KEY`. +#### GitPython + +The default name for the ssh key is `gitfs_ssh` but it can be changed with the env variables `SALT_GITFS_SSH_PRIVATE_KEY` and `SALT_GITFS_SSH_PUBLIC_KEY`. This keys must be placed inside `/home/salt/data/keys` directory. +#### PyGit2 + +You can create an ssh key for pygit2 with the following command: + +```sh +ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com' +``` + +Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file. + +For example: + +```yml +gitfs_provider: pygit2 +gitfs_privkey: /home/salt/data/keys/gitfs/gitfs_ssh +gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub +``` + +**Important Note** + +If you get the following error while using `gitfs` with `pygit2` + +```plain +_pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request +``` + +look if your private key hash empty lines at the bottom of the file and suppress them for solving the error. + ### Available Configuration Parameters Please refer the docker run command options for the `--env-file` flag where you can specify all required environment variables in a single file. This will save you from writing a potentially long docker run command. Alternatively you can use docker-compose.