Merge pull request #6 from cdalvaro/feature/pygit2_support

Add support for PyGit2

Dockerfile

@@ -4,6 +4,13 @@ LABEL maintainer="carlos.alvaro@citelan.es"
 LABEL description="SaltStack master"
 LABEL version="2018.3.3"
 
+# https://github.com/saltstack/salt/releases
+ENV SALT_VERSION="2018.3.3" \
+    PYTHON_VERSION="3.5" \
+    LIBSSH2_VERSION="1.8.0" \
+    LIBGIT2_VERSION="0.27.7" \
+    PYGIT2_VERSION="0.27.2"
+
 ENV SALT_DOCKER_DIR="/etc/docker-salt" \
     SALT_ROOT_DIR="/etc/salt" \
     SALT_USER="salt" \
@@ -17,28 +24,18 @@ ENV SALT_CONFS_DIR="${SALT_DATA_DIR}/config" \
     SALT_KEYS_DIR="${SALT_DATA_DIR}/keys" \
     SALT_BASE_DIR="${SALT_DATA_DIR}/srv"
 
-# Bootstrap script options:
-# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options
-## -M: install Salt Master by default
-## -N: Do not install salt-minion
-## -X: Do not start daemons after installation
-## -U: Fully upgrade the system prior to bootstrapping Salt
-## -V: Install Salt into virtualenv
-## -a: Pip install all Python pkg dependencies for Salt
-ENV SALT_BOOTSTRAP_OPTS='-M -N -X -U -Va'
-
-# Release version to install
-# https://github.com/saltstack/salt/releases
-ENV SALT_GIT_RELEASE="v2018.3.3"
-
 # Set non interactive mode
 ENV DEBIAN_FRONTEND=noninteractive
 
+RUN mkdir -p ${SALT_BUILD_DIR}
+WORKDIR ${SALT_BUILD_DIR}
+
 # Install packages
 RUN apt-get update
 RUN apt-get install --yes --quiet --no-install-recommends \
-    ca-certificates apt-transport-https curl git vim locales \
+    ca-certificates wget locales pkg-config openssh-client \
-    openssh-client python3 python-git
+    python${PYTHON_VERSION} python${PYTHON_VERSION}-dev \
+    python3-pip python3-setuptools python3-wheel
 
 # Configure locales
 RUN update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \
@@ -46,16 +43,10 @@ RUN update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \
     dpkg-reconfigure locales
 
 # Install saltstack
-RUN mkdir -p ${SALT_BUILD_DIR}
+COPY assets/build ${SALT_BUILD_DIR}
-WORKDIR ${SALT_BUILD_DIR}
+RUN bash ${SALT_BUILD_DIR}/install.sh
-
-RUN curl -o bootstrap-salt.sh -L https://bootstrap.saltstack.com
-RUN sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} git ${SALT_GIT_RELEASE}
-
-# Salt user
-RUN useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER}
-RUN chown -R ${SALT_USER}: ${SALT_ROOT_DIR}
 
+# Shared resources
 EXPOSE 4505/tcp 4506/tcp
 RUN mkdir -p ${SALT_DATA_DIR} ${SALT_BASE_DIR} ${SALT_KEYS_DIR} ${SALT_CONFS_DIR}
 VOLUME [ "${SALT_BASE_DIR}" "${SALT_KEYS_DIR}" "${SALT_CONFS_DIR}" ]
@@ -66,6 +57,7 @@ RUN chmod -R +x ${SALT_RUNTIME_DIR}
 # Cleaning tasks
 RUN apt-get clean --yes
 RUN rm -rf /var/lib/apt/lists/*
+RUN rm -rf ${SALT_BUILD_DIR}/*
 
 # Entrypoint
 COPY entrypoint.sh /sbin/entrypoint.sh

README.md

@@ -17,6 +17,8 @@ For other methods to install SaltStack please refer to the [Official SaltStack I
   - [Master Signed Keys](#master-signed-keys)
   - [Host Mapping](#host-mapping)
   - [Git Fileserver](#git-fileserver)
+    - [GitPython](#gitpython)
+    - [PyGit2](#pygit2)
   - [Available Configuration Parameters](#available-configuration-parameters)
 - [Usage](#usage)
 - [Shell Access](#shell-access)
@@ -131,14 +133,44 @@ docker run --name salt_stack -it --rm \
 
 ### Git Fileserver
 
-This image uses [GitPython](https://github.com/gitpython-developers/GitPython) as gitfs backend to allow Salt to serve files from git repositories.
+This image uses [GitPython](https://github.com/gitpython-developers/GitPython) and [PyGit2](https://www.pygit2.org) as gitfs backends to allow Salt to serve files from git repositories.
 
 It can be enabled by adding `gitfs` to the [`fileserver_backend`](https://docs.saltstack.com/en/latest/ref/configuration/master.html#std:conf_master-fileserver_backend) list (see [Available Configuration Parameters](#available-configuration-parameters)), and configuring one or more repositories in [`gitfs_remotes`](https://docs.saltstack.com/en/latest/ref/configuration/master.html#std:conf_master-gitfs_remotes).
 
-As the backend for gitfs is GitPython, then an ssh key is needed. The default name for the ssh key is `gitfs_ssh` but it can be changed with the env variables `SALT_GITFS_SSH_PRIVATE_KEY` and `SALT_GITFS_SSH_PUBLIC_KEY`.
+#### GitPython
+
+The default name for the ssh key is `gitfs_ssh` but it can be changed with the env variables `SALT_GITFS_SSH_PRIVATE_KEY` and `SALT_GITFS_SSH_PUBLIC_KEY`.
 
 This keys must be placed inside `/home/salt/data/keys` directory.
 
+#### PyGit2
+
+You can create an ssh key for pygit2 with the following command:
+
+```sh
+ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com'
+```
+
+Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey`  and `gitfs_privkey`  in your `.conf` file.
+
+For example:
+
+```yml
+gitfs_provider: pygit2
+gitfs_privkey: /home/salt/data/keys/gitfs/gitfs_ssh
+gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub
+```
+
+**Important Note** 
+
+If you get the following error while using `gitfs` with `pygit2`
+
+```plain
+_pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request
+```
+
+look if your private key hash empty lines at the bottom of the file and suppress them for solving the error.
+
 ### Available Configuration Parameters
 
 Please refer the docker run command options for the `--env-file` flag where you can specify all required environment variables in a single file. This will save you from writing a potentially long docker run command. Alternatively you can use docker-compose.

assets/build/install.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Install build dependencies
+echo "Installing dependencies ..."
+BUILD_DEPENDENCIES="gnupg git cmake gcc g++ make \
+    libhttp-parser-dev libssl-dev zlib1g-dev \
+    libzmq-dev libcurl4-openssl-dev libffi-dev"
+
+apt-get install --yes --quiet --no-install-recommends ${BUILD_DEPENDENCIES}
+
+# Compile libssh2
+echo "Building libssh2 v${LIBSSH2_VERSION} ..."
+wget https://github.com/libssh2/libssh2/archive/libssh2-${LIBSSH2_VERSION}.tar.gz 
+tar xzf libssh2-${LIBSSH2_VERSION}.tar.gz
+cd libssh2-libssh2-${LIBSSH2_VERSION}/
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DENABLE_ZLIB_COMPRESSION=ON .
+cmake --build . --target install
+
+# Compile libgit2
+echo "Building libgit2 v${LIBGIT2_VERSION} ..."
+wget https://github.com/libgit2/libgit2/archive/v${LIBGIT2_VERSION}.tar.gz
+tar xzf v${LIBGIT2_VERSION}.tar.gz
+cd libgit2-${LIBGIT2_VERSION}/
+cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DBUILD_CLAR=OFF -DTHREADSAFE=ON .
+cmake --build . --target install
+
+# Install python packages
+echo "Installing python packages ..."
+pip3 install "pygit2==v${PYGIT2_VERSION}" gitpython
+
+# Salt user
+echo "Creating ${SALT_USER} user ..."
+useradd -d ${SALT_HOME} -ms /bin/bash -U -G root,sudo ${SALT_USER}
+
+# Bootstrap script options:
+# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options
+## -M: install Salt Master by default
+## -N: Do not install salt-minion
+## -X: Do not start daemons after installation
+## -P: Allow pip based installations
+## -x: Changes the python version used to install a git version of salt
+SALT_BOOTSTRAP_OPTS="-M -N -X -P -x python${PYTHON_VERSION}"
+
+echo "Installing saltstack ..."
+wget -O bootstrap-salt.sh https://bootstrap.saltstack.com
+sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS} stable ${SALT_VERSION}
+chown -R ${SALT_USER}: ${SALT_ROOT_DIR}

assets/runtime/functions.sh

@@ -64,7 +64,7 @@ EOF
     salt-key --gen-signature --auto-create --pub ${SALT_KEYS_DIR}/master.pub --signature-path ${SALT_KEYS_DIR}
   fi
 
-  for pub_key in $(find ${SALT_KEYS_DIR} -type f -maxdepth 2); do
+  for pub_key in $(find ${SALT_KEYS_DIR} -type f -maxdepth 1); do
     if [[ ${pub_key} =~ .*\.pem$ ]]; then
       chmod 400 ${pub_key}
     else
@@ -72,6 +72,7 @@ EOF
     fi
   done
 
+  find ${SALT_KEYS_DIR}/minions* -type f -maxdepth 1 -exec chmod 644 {} \;
   find ${SALT_HOME} -path ${SALT_KEYS_DIR}/\* -prune -o -print0 | xargs -0 chown -h ${SALT_USER}:
 }