From d795ea2963a05acbf2f82f78621311e7a561274f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20=C3=81lvaro?= Date: Sun, 31 Oct 2021 22:17:22 +0100 Subject: [PATCH] feat: Install pygit2 1.7.0 --- .github/workflows/build-and-test.yml | 9 ++ .gitignore | 8 +- CHANGELOG.md | 6 + Dockerfile | 2 +- README.md | 10 +- assets/build/functions.sh | 224 ++++++++++++++++++++++++++- assets/build/install.sh | 81 ++++++---- tests/gitfs/README.md | 4 + tests/gitfs/config/fileserver.conf | 2 + tests/gitfs/config/gitfs.conf | 19 +++ tests/gitfs/test.sh | 41 +++++ tests/salt-api/test.sh | 26 ++-- 12 files changed, 381 insertions(+), 51 deletions(-) create mode 100644 tests/gitfs/README.md create mode 100644 tests/gitfs/config/fileserver.conf create mode 100644 tests/gitfs/config/gitfs.conf create mode 100755 tests/gitfs/test.sh diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ed0685d..7d74b68 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -119,6 +119,15 @@ jobs: - name: Execute salt-api tests run: tests/salt-api/test.sh + - name: Execute gitfs tests + env: + GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs + run: | + mkdir -p "${GITFS_KEYS_DIR}" + echo ${{ secrets.TESTS_REPO_PRIVATE_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$ + echo ${{ secrets.TESTS_REPO_PUBLIC_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$ + tests/gitfs/test.sh + - name: Cleanup run: | docker stop registry diff --git a/.gitignore b/.gitignore index d70f409..8c922aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,13 @@ +# Always included +!assets/** + # development /config/ /roots/ /keys/ /logs/ /3pfs/ -.vscode/ \ No newline at end of file +.vscode/ + +# tests +/tests/**/keys/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b01faa7..72be1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ This file only reflects the changes that are made in this image. Please refer to the [Salt 3004 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3004.html) for the list of changes in SaltStack. +**3004_1** + +- Install `libssh2 1.10.0` from source +- Install `libgit2 1.3.0` from source +- Install `pygit2 1.7.0` from pip repositories + **3004** - Upgrade `salt-master` to `3004` *Silicon* diff --git a/Dockerfile b/Dockerfile index 6d3582e..b3bedcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ WORKDIR ${SALT_BUILD_DIR} # hadolint ignore=DL3008 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - sudo ca-certificates apt-transport-https wget locales openssh-client \ + sudo ca-certificates openssl apt-transport-https wget locales openssh-client \ python${PYTHON_VERSION} python3-dev libpython3-dev \ python3-pip python3-setuptools python3-wheel \ supervisor logrotate git gettext-base tzdata \ diff --git a/README.md b/README.md index fff4053..5bb017a 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ This keys must be placed inside `/home/salt/data/keys` directory. You can create an ssh key for pygit2 with the following command: ```sh -ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com' +ssh-keygen -f gitfs_ssh -C 'gitfs@example.com' ``` Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file. @@ -334,13 +334,19 @@ gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub **Important Note** +By default, this image has been tested with RSA 4096 ssh keys generated with `ssh-keygen`. + If you get the following error while using `gitfs` with `pygit2` ```plain _pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request ``` -look if your private key hash empty lines at the bottom of the file and suppress them for solving the error. +you may have to recreate your ssh key adding the parameter: `-m PEM`: + +```sh +ssh-keygen -m PEM -f gitfs_ssh -C 'gitfs@example.com' +``` ### 3rd Party Formulas diff --git a/assets/build/functions.sh b/assets/build/functions.sh index 300ddea..8254e0f 100755 --- a/assets/build/functions.sh +++ b/assets/build/functions.sh @@ -2,7 +2,68 @@ set -e -# Execute a command as SALT_USER +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __detect_color_support +# DESCRIPTION: Try to detect color support. +#---------------------------------------------------------------------------------------------------------------------- +_COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} +__detect_color_support() { + # shellcheck disable=SC2181 + if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then + RC='\033[1;31m' + GC='\033[1;32m' + BC='\033[1;34m' + YC='\033[1;33m' + EC='\033[0m' + else + RC="" + GC="" + BC="" + YC="" + EC="" + fi +} +__detect_color_support + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_debug +# DESCRIPTION: Echo debug information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_debug() { + if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then + echo -e "${BC} * DEBUG${EC}: $*" + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_info +# DESCRIPTION: Echo information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_info() { + echo -e "${GC} * INFO${EC}: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_warn +# DESCRIPTION: Echo warning information to stdout. +#---------------------------------------------------------------------------------------------------------------------- +function log_warn() { + echo -e "${YC} * WARN${EC}: $*" +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: log_error +# DESCRIPTION: Echo errors to stderr. +#---------------------------------------------------------------------------------------------------------------------- +function log_error() +{ + (>&2 echo -e "${RC} * ERROR${EC}: $*") +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: exec_as_salt +# DESCRIPTION: Execute the pass command as the `salt` user. +#---------------------------------------------------------------------------------------------------------------------- function exec_as_salt() { if [[ $(whoami) == "${SALT_USER}" ]]; then @@ -11,3 +72,164 @@ function exec_as_salt() sudo -HEu "${SALT_USER}" "$@" fi } + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 32-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm32() +{ + uname -m | grep -qE 'armv7l' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM 64-bits or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm64() +{ + uname -m | grep -qE 'arm64|aarch64' +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: is_arm32 +# DESCRIPTION: Check whether the platform is ARM or not. +#---------------------------------------------------------------------------------------------------------------------- +function is_arm() +{ + is_arm32 || is_arm64 +} + +function install_pkgs() +{ + apt-get install --no-install-recommends -y $@ +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: download +# DESCRIPTION: Download the content from the given URL and save it into the specified file. +#---------------------------------------------------------------------------------------------------------------------- +function download() +{ + local URL="$1" + local FILE_NAME="$2" + + local WGET_ARGS=(--quiet) + is_arm32 && WGET_ARGS+=(--no-check-certificate) + + log_info "Downloading ${FILE_NAME} from ${URL} ..." + wget ${WGET_ARGS[@]} -O "${FILE_NAME}" "${URL}" + if [[ -f "${FILE_NAME}" ]]; then + log_debug "Success!" + else + log_error "Failed to download ${URL}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: check_sha256 +# DESCRIPTION: Compute the SHA256 hash for the given file and check if it matches the expected one. +#---------------------------------------------------------------------------------------------------------------------- +function check_sha256() +{ + local FILE="${1}" + local SHA256="${2}" + + log_info "Checking ${FILE} SHA256 hash ..." + if echo "${SHA256} ${FILE}" | shasum -a 256 -c --status -; then + log_debug "SHA256 hash for ${FILE} matches! (${SHA256})" + else + local HASH=$(shasum -a 256 "${FILE}" | awk '{print $1}') + log_error "SHA256 checksum mismatch for ${FILE}" + log_error "Expected: ${SHA256}" + log_error " Got: ${HASH}" + exit 1 + fi +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: extract +# DESCRIPTION: Extract the given .tar.gz into the current directory. +#---------------------------------------------------------------------------------------------------------------------- +function extract() +{ + local FILE="${1}" + log_info "Unpacking file: ${FILE}" + tar xzf "${FILE}" --strip-components 1 +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: build_and_install +# DESCRIPTION: Build and install the given package from the current directory using cmake. +#---------------------------------------------------------------------------------------------------------------------- +function build_and_install() +{ + local PACKAGE_NAME="${1}"; shift + local CMAKE_ARGS=( + -Wno-dev + -DCMAKE_BUILD_TYPE=Release + ) + + # shellcheck disable=SC2206 + CMAKE_ARGS+=( $@ ) + + log_info "Building and installing ${PACKAGE_NAME} ..." + log_debug "CMAKE_ARGS: ${CMAKE_ARGS[@]}" + cmake ${CMAKE_ARGS[@]} . + cmake --build . --target install --config Release +} + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: install_libssh2 +# DESCRIPTION: Install libssh2 library. +#---------------------------------------------------------------------------------------------------------------------- +function install_libssh2() +{ + local LIBSSH2_VERSION=1.10.0 + local LIBSSH2_URL="https://github.com/libssh2/libssh2/releases/download/libssh2-${LIBSSH2_VERSION}/libssh2-${LIBSSH2_VERSION}.tar.gz" + local FILE_NAME="libssh2-${LIBSSH2_VERSION}.tar.gz" + local SHA256_SUM='2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBSSH2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + _OPTS=( + -DLINT=OFF + -DBUILD_SHARED_LIBS=ON + -DCRYPTO_BACKEND=OpenSSL + -DENABLE_ZLIB_COMPRESSION=ON + -DENABLE_DEBUG_LOGGING=OFF + -DCLEAR_MEMORY=ON + ) + + build_and_install "libssh2 v${LIBSSH2_VERSION}" ${_OPTS[@]} + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} + +# Install libgit2 library +function install_libgit2() +{ + local LIBGIT2_VERSION=1.3.0 + local LIBGIT2_URL="https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz" + local FILE_NAME="libgit2-${LIBGIT2_VERSION}.tar.gz" + local SHA256_SUM='192eeff84596ff09efb6b01835a066f2df7cd7985e0991c79595688e6b36444e' + + local CURRENT_DIR="$(pwd)" + local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}" + + download "${LIBGIT2_URL}" "${FILE_NAME}" + check_sha256 "${FILE_NAME}" "${SHA256_SUM}" + extract "${FILE_NAME}" + + build_and_install "libgit2 v${LIBGIT2_VERSION}" -DBUILD_CLAR=OFF + + cd "${CURRENT_DIR}" + rm -rf "${WORK_DIR}" +} diff --git a/assets/build/install.sh b/assets/build/install.sh index 3ca5994..c47e71a 100755 --- a/assets/build/install.sh +++ b/assets/build/install.sh @@ -2,24 +2,28 @@ set -e -# shellcheck disable=SC1091 -source "${SALT_BUILD_DIR}/functions.sh" +export DEBIAN_FRONTEND=noninteractive -echo "Installing build dependencies ..." -BUILD_DEPENDENCIES=(make gcc g++ cmake pkg-config) +# shellcheck source=assets/build/functions.sh +FUNCTIONS_FILE="${SALT_BUILD_DIR}/functions.sh" +source "${FUNCTIONS_FILE}" + +log_info "Installing required packages and build dependencies ..." +REQUIRED_PACKAGES=( + libssl1.1 zlib1g libffi7 libpcre3 libgssapi3-heimdal +) + +BUILD_DEPENDENCIES=( + make gcc g++ cmake pkg-config libssl-dev zlib1g-dev libffi-dev + libpcre3-dev heimdal-dev +) apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y "${BUILD_DEPENDENCIES[@]}" - -# Install arm build dependencies -if [[ "$(uname -i)" =~ ^(arm|aarch64) ]]; then - echo "Installing arm specific dependencies ..." - DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - libzmq3-dev libhttp-parser-dev libssl-dev libcurl4-openssl-dev -fi +apt-get install --no-install-recommends -y \ + "${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}" # Create salt user -echo "Creating ${SALT_USER} user ..." +log_info "Creating ${SALT_USER} user ..." useradd -d "${SALT_HOME}" -ms /bin/bash -U -G root,sudo,shadow "${SALT_USER}" # Set PATH @@ -28,14 +32,28 @@ PATH=/usr/local/sbin:/usr/local/bin:\$PATH EOF # Install python3 packages -echo "Installing python3 packages ..." -DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \ - python3-mako python3-pycryptodome python3-cherrypy3 python3-git python3-requests \ - python3-redis python3-gnupg python3-mysqldb python3-dateutil python3-libnacl python3-openssl \ - python3-pygit2 +log_info "Installing python3 packages ..." +apt-get install --yes --quiet --no-install-recommends \ + python3-mako python3-pycryptodome python3-cherrypy3 \ + python3-git python3-requests python3-redis python3-gnupg \ + python3-mysqldb python3-dateutil python3-libnacl python3-openssl pip3 install timelib==0.2.5 +# Install pygit2 package +install_libssh2 +install_libgit2 +pip3 install pygit2==1.7.0 + +# Downloading bootstrap-salt.sh script +BOOTSTRAP_VERSION='2021.09.17' +BOOTSTRAP_URL="https://raw.githubusercontent.com/saltstack/salt-bootstrap/v${BOOTSTRAP_VERSION}/bootstrap-salt.sh" +BOOTSTRAP_FILE='bootstrap-salt.sh' +BOOTSTRAP_SHA256='090d652cd6290debce0e3a4eded65086a4272e69446e711eb26f87160593b6a2' + +download "${BOOTSTRAP_URL}" "${BOOTSTRAP_FILE}" +check_sha256 "${BOOTSTRAP_FILE}" "${BOOTSTRAP_SHA256}" + # Bootstrap script options: # https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options ## -M: install Salt Master by default @@ -46,22 +64,17 @@ pip3 install timelib==0.2.5 ## -p: Extra-package to install ## -x: Changes the python version used to install a git version of salt SALT_BOOTSTRAP_OPTS=( -M -N -X -d -P -p salt-api -p salt-call -x "python${PYTHON_VERSION}" ) -_WGET_ARGS=() -if [[ "$(uname -i)" == 'armv7l' ]]; then - ## -I: allow insecure connections while downloading any files - SALT_BOOTSTRAP_OPTS+=( -I ) - _WGET_ARGS+=( --no-check-certificate ) -fi +## -I: allow insecure connections while downloading any files +is_arm32 && SALT_BOOTSTRAP_OPTS+=( -I ) -echo "Installing saltstack ..." -echo "Option: ${SALT_BOOTSTRAP_OPTS[@]}" -wget ${_WGET_ARGS[@]} -O bootstrap-salt.sh https://bootstrap.saltstack.com -sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" +log_info "Installing saltstack ..." +log_debug "Options: ${SALT_BOOTSTRAP_OPTS[@]}" +sh "${BOOTSTRAP_FILE}" ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}" chown -R "${SALT_USER}": "${SALT_ROOT_DIR}" # Configure ssh -echo "Configuring ssh ..." +log_info "Configuring ssh ..." sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ssh/ssh_config { echo " UserKnownHostsFile /dev/null" @@ -70,7 +83,7 @@ sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ } >> /etc/ssh/ssh_config # Configure logrotate -echo "Configuring logrotate ..." +log_info "Configuring logrotate ..." # move supervisord.log file to ${SALT_LOGS_DIR}/supervisor/ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;|" /etc/supervisor/supervisord.conf @@ -79,7 +92,7 @@ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;| sed -i "s|^su root syslog$|su root root|" /etc/logrotate.conf # Configure supervisor -echo "Configuring supervisor ..." +log_info "Configuring supervisor ..." # configure supervisord to start salt-master cat > /etc/supervisor/conf.d/salt-master.conf < Removing ${CONTAINER_NAME} ..." + docker container rm --force "${CONTAINER_NAME}" +} + +# trap cleanup EXIT + +# Run test instance +echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..." +docker run --rm --detach --name "${CONTAINER_NAME}" \ + --publish 4505:4505 --publish 4506:4506 \ + --platform "${PLATFORM}" \ + --volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \ + --volume "$(pwd)/tests/gitfs/data/keys":/home/salt/data/keys \ + "${IMAGE_NAME}" || ( echo "container started โŒ"; exit 1 ) +echo "container started โœ…" + +# Wait for salt-master bootup +echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." +sleep "${BOOTUP_WAIT_SECONDS}" + +# Check pillars +echo "==> Checking gitfs files ..." +# docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update +# docker exec "${CONTAINER_NAME}" salt-run fileserver.update +FILE_LIST=$(docker exec "${CONTAINER_NAME}" salt-run fileserver.file_list) +echo "${FILE_LIST}" +[[ "${FILE_LIST}" == *test.txt* ]] || ( echo "gitfs files โŒ"; exit 1 ) +echo "gitfs files โœ…" diff --git a/tests/salt-api/test.sh b/tests/salt-api/test.sh index aef7731..4ab13be 100755 --- a/tests/salt-api/test.sh +++ b/tests/salt-api/test.sh @@ -52,24 +52,24 @@ echo "container started โœ…" echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..." sleep "${BOOTUP_WAIT_SECONDS}" - # Test salt-api authentication - echo "==> Getting salt-api token ..." +# Test salt-api authentication +echo "==> Getting salt-api token ..." SALTAPI_TOKEN=$(curl -sSk "${SALTAPI_URL%/}/login" \ - -H "Accept: application/x-yaml" \ - -d username="${SALTAPI_USER}" \ - -d password="${SALTAPI_PASS}" \ - -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) + -H "Accept: application/x-yaml" \ + -d username="${SALTAPI_USER}" \ + -d password="${SALTAPI_PASS}" \ + -d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4) [ -n "${SALTAPI_TOKEN}" ] || ( echo "salt-api token โŒ"; exit 1 ) echo "salt-api token โœ…" - # Test salt-api command - echo "==> Testing curl command ..." +# Test salt-api command +echo "==> Testing curl command ..." curl -sSk "${SALTAPI_URL}" \ - -H "Accept: application/x-yaml" \ - -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ - -d client=runner \ - -d tgt='*' \ - -d fun=test.stream \ + -H "Accept: application/x-yaml" \ + -H "X-Auth-Token: ${SALTAPI_TOKEN}" \ + -d client=runner \ + -d tgt='*' \ + -d fun=test.stream \ | grep -i 'true' || ( echo "curl command โŒ"; exit 1 ) echo "curl command โœ…"