feat: Install pygit2 1.7.0

This commit is contained in:
Carlos Álvaro
2021-10-31 22:17:22 +01:00
committed by Carlos Álvaro
parent c8370829d0
commit d795ea2963
12 changed files with 381 additions and 51 deletions

View File

@@ -119,6 +119,15 @@ jobs:
- name: Execute salt-api tests
run: tests/salt-api/test.sh
- name: Execute gitfs tests
env:
GITFS_KEYS_DIR: tests/gitfs/data/keys/gitfs
run: |
mkdir -p "${GITFS_KEYS_DIR}"
echo ${{ secrets.TESTS_REPO_PRIVATE_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh && chmod 600 !$
echo ${{ secrets.TESTS_REPO_PUBLIC_KEY }} > "${GITFS_KEYS_DIR}"/gitfs_ssh.pub && chmod 644 !$
tests/gitfs/test.sh
- name: Cleanup
run: |
docker stop registry

8
.gitignore vendored
View File

@@ -1,7 +1,13 @@
# Always included
!assets/**
# development
/config/
/roots/
/keys/
/logs/
/3pfs/
.vscode/
.vscode/
# tests
/tests/**/keys/

View File

@@ -4,6 +4,12 @@ This file only reflects the changes that are made in this image.
Please refer to the [Salt 3004 Release Notes](https://docs.saltstack.com/en/latest/topics/releases/3004.html)
for the list of changes in SaltStack.
**3004_1**
- Install `libssh2 1.10.0` from source
- Install `libgit2 1.3.0` from source
- Install `pygit2 1.7.0` from pip repositories
**3004**
- Upgrade `salt-master` to `3004` *Silicon*

View File

@@ -32,7 +32,7 @@ WORKDIR ${SALT_BUILD_DIR}
# hadolint ignore=DL3008
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
sudo ca-certificates apt-transport-https wget locales openssh-client \
sudo ca-certificates openssl apt-transport-https wget locales openssh-client \
python${PYTHON_VERSION} python3-dev libpython3-dev \
python3-pip python3-setuptools python3-wheel \
supervisor logrotate git gettext-base tzdata \

View File

@@ -319,7 +319,7 @@ This keys must be placed inside `/home/salt/data/keys` directory.
You can create an ssh key for pygit2 with the following command:
```sh
ssh-keygen -f gitfs_pygit2 -C 'gitfs@example.com'
ssh-keygen -f gitfs_ssh -C 'gitfs@example.com'
```
Place it wherever you want inside the container and specify its path with the configuration parameters: `gitfs_pubkey` and `gitfs_privkey` in your `.conf` file.
@@ -334,13 +334,19 @@ gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub
**Important Note**
By default, this image has been tested with RSA 4096 ssh keys generated with `ssh-keygen`.
If you get the following error while using `gitfs` with `pygit2`
```plain
_pygit2.GitError: Failed to authenticate SSH session: Unable to send userauth-publickey request
```
look if your private key hash empty lines at the bottom of the file and suppress them for solving the error.
you may have to recreate your ssh key adding the parameter: `-m PEM`:
```sh
ssh-keygen -m PEM -f gitfs_ssh -C 'gitfs@example.com'
```
### 3rd Party Formulas

View File

@@ -2,7 +2,68 @@
set -e
# Execute a command as SALT_USER
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: __detect_color_support
# DESCRIPTION: Try to detect color support.
#----------------------------------------------------------------------------------------------------------------------
_COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)}
__detect_color_support() {
# shellcheck disable=SC2181
if [ $? -eq 0 ] && [ "$_COLORS" -gt 2 ]; then
RC='\033[1;31m'
GC='\033[1;32m'
BC='\033[1;34m'
YC='\033[1;33m'
EC='\033[0m'
else
RC=""
GC=""
BC=""
YC=""
EC=""
fi
}
__detect_color_support
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: log_debug
# DESCRIPTION: Echo debug information to stdout.
#----------------------------------------------------------------------------------------------------------------------
function log_debug() {
if [[ "${DEBUG}" == 'true' || "${ECHO_DEBUG}" == 'true' ]]; then
echo -e "${BC} * DEBUG${EC}: $*"
fi
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: log_info
# DESCRIPTION: Echo information to stdout.
#----------------------------------------------------------------------------------------------------------------------
function log_info() {
echo -e "${GC} * INFO${EC}: $*"
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: log_warn
# DESCRIPTION: Echo warning information to stdout.
#----------------------------------------------------------------------------------------------------------------------
function log_warn() {
echo -e "${YC} * WARN${EC}: $*"
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: log_error
# DESCRIPTION: Echo errors to stderr.
#----------------------------------------------------------------------------------------------------------------------
function log_error()
{
(>&2 echo -e "${RC} * ERROR${EC}: $*")
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: exec_as_salt
# DESCRIPTION: Execute the pass command as the `salt` user.
#----------------------------------------------------------------------------------------------------------------------
function exec_as_salt()
{
if [[ $(whoami) == "${SALT_USER}" ]]; then
@@ -11,3 +72,164 @@ function exec_as_salt()
sudo -HEu "${SALT_USER}" "$@"
fi
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: is_arm32
# DESCRIPTION: Check whether the platform is ARM 32-bits or not.
#----------------------------------------------------------------------------------------------------------------------
function is_arm32()
{
uname -m | grep -qE 'armv7l'
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: is_arm32
# DESCRIPTION: Check whether the platform is ARM 64-bits or not.
#----------------------------------------------------------------------------------------------------------------------
function is_arm64()
{
uname -m | grep -qE 'arm64|aarch64'
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: is_arm32
# DESCRIPTION: Check whether the platform is ARM or not.
#----------------------------------------------------------------------------------------------------------------------
function is_arm()
{
is_arm32 || is_arm64
}
function install_pkgs()
{
apt-get install --no-install-recommends -y $@
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: download
# DESCRIPTION: Download the content from the given URL and save it into the specified file.
#----------------------------------------------------------------------------------------------------------------------
function download()
{
local URL="$1"
local FILE_NAME="$2"
local WGET_ARGS=(--quiet)
is_arm32 && WGET_ARGS+=(--no-check-certificate)
log_info "Downloading ${FILE_NAME} from ${URL} ..."
wget ${WGET_ARGS[@]} -O "${FILE_NAME}" "${URL}"
if [[ -f "${FILE_NAME}" ]]; then
log_debug "Success!"
else
log_error "Failed to download ${URL}"
exit 1
fi
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: check_sha256
# DESCRIPTION: Compute the SHA256 hash for the given file and check if it matches the expected one.
#----------------------------------------------------------------------------------------------------------------------
function check_sha256()
{
local FILE="${1}"
local SHA256="${2}"
log_info "Checking ${FILE} SHA256 hash ..."
if echo "${SHA256} ${FILE}" | shasum -a 256 -c --status -; then
log_debug "SHA256 hash for ${FILE} matches! (${SHA256})"
else
local HASH=$(shasum -a 256 "${FILE}" | awk '{print $1}')
log_error "SHA256 checksum mismatch for ${FILE}"
log_error "Expected: ${SHA256}"
log_error " Got: ${HASH}"
exit 1
fi
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: extract
# DESCRIPTION: Extract the given .tar.gz into the current directory.
#----------------------------------------------------------------------------------------------------------------------
function extract()
{
local FILE="${1}"
log_info "Unpacking file: ${FILE}"
tar xzf "${FILE}" --strip-components 1
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: build_and_install
# DESCRIPTION: Build and install the given package from the current directory using cmake.
#----------------------------------------------------------------------------------------------------------------------
function build_and_install()
{
local PACKAGE_NAME="${1}"; shift
local CMAKE_ARGS=(
-Wno-dev
-DCMAKE_BUILD_TYPE=Release
)
# shellcheck disable=SC2206
CMAKE_ARGS+=( $@ )
log_info "Building and installing ${PACKAGE_NAME} ..."
log_debug "CMAKE_ARGS: ${CMAKE_ARGS[@]}"
cmake ${CMAKE_ARGS[@]} .
cmake --build . --target install --config Release
}
#--- FUNCTION -------------------------------------------------------------------------------------------------------
# NAME: install_libssh2
# DESCRIPTION: Install libssh2 library.
#----------------------------------------------------------------------------------------------------------------------
function install_libssh2()
{
local LIBSSH2_VERSION=1.10.0
local LIBSSH2_URL="https://github.com/libssh2/libssh2/releases/download/libssh2-${LIBSSH2_VERSION}/libssh2-${LIBSSH2_VERSION}.tar.gz"
local FILE_NAME="libssh2-${LIBSSH2_VERSION}.tar.gz"
local SHA256_SUM='2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51'
local CURRENT_DIR="$(pwd)"
local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}"
download "${LIBSSH2_URL}" "${FILE_NAME}"
check_sha256 "${FILE_NAME}" "${SHA256_SUM}"
extract "${FILE_NAME}"
_OPTS=(
-DLINT=OFF
-DBUILD_SHARED_LIBS=ON
-DCRYPTO_BACKEND=OpenSSL
-DENABLE_ZLIB_COMPRESSION=ON
-DENABLE_DEBUG_LOGGING=OFF
-DCLEAR_MEMORY=ON
)
build_and_install "libssh2 v${LIBSSH2_VERSION}" ${_OPTS[@]}
cd "${CURRENT_DIR}"
rm -rf "${WORK_DIR}"
}
# Install libgit2 library
function install_libgit2()
{
local LIBGIT2_VERSION=1.3.0
local LIBGIT2_URL="https://github.com/libgit2/libgit2/archive/refs/tags/v${LIBGIT2_VERSION}.tar.gz"
local FILE_NAME="libgit2-${LIBGIT2_VERSION}.tar.gz"
local SHA256_SUM='192eeff84596ff09efb6b01835a066f2df7cd7985e0991c79595688e6b36444e'
local CURRENT_DIR="$(pwd)"
local WORK_DIR="$(mktemp -d)" && cd "${WORK_DIR}"
download "${LIBGIT2_URL}" "${FILE_NAME}"
check_sha256 "${FILE_NAME}" "${SHA256_SUM}"
extract "${FILE_NAME}"
build_and_install "libgit2 v${LIBGIT2_VERSION}" -DBUILD_CLAR=OFF
cd "${CURRENT_DIR}"
rm -rf "${WORK_DIR}"
}

View File

@@ -2,24 +2,28 @@
set -e
# shellcheck disable=SC1091
source "${SALT_BUILD_DIR}/functions.sh"
export DEBIAN_FRONTEND=noninteractive
echo "Installing build dependencies ..."
BUILD_DEPENDENCIES=(make gcc g++ cmake pkg-config)
# shellcheck source=assets/build/functions.sh
FUNCTIONS_FILE="${SALT_BUILD_DIR}/functions.sh"
source "${FUNCTIONS_FILE}"
log_info "Installing required packages and build dependencies ..."
REQUIRED_PACKAGES=(
libssl1.1 zlib1g libffi7 libpcre3 libgssapi3-heimdal
)
BUILD_DEPENDENCIES=(
make gcc g++ cmake pkg-config libssl-dev zlib1g-dev libffi-dev
libpcre3-dev heimdal-dev
)
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y "${BUILD_DEPENDENCIES[@]}"
# Install arm build dependencies
if [[ "$(uname -i)" =~ ^(arm|aarch64) ]]; then
echo "Installing arm specific dependencies ..."
DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
libzmq3-dev libhttp-parser-dev libssl-dev libcurl4-openssl-dev
fi
apt-get install --no-install-recommends -y \
"${REQUIRED_PACKAGES[@]}" "${BUILD_DEPENDENCIES[@]}"
# Create salt user
echo "Creating ${SALT_USER} user ..."
log_info "Creating ${SALT_USER} user ..."
useradd -d "${SALT_HOME}" -ms /bin/bash -U -G root,sudo,shadow "${SALT_USER}"
# Set PATH
@@ -28,14 +32,28 @@ PATH=/usr/local/sbin:/usr/local/bin:\$PATH
EOF
# Install python3 packages
echo "Installing python3 packages ..."
DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet --no-install-recommends \
python3-mako python3-pycryptodome python3-cherrypy3 python3-git python3-requests \
python3-redis python3-gnupg python3-mysqldb python3-dateutil python3-libnacl python3-openssl \
python3-pygit2
log_info "Installing python3 packages ..."
apt-get install --yes --quiet --no-install-recommends \
python3-mako python3-pycryptodome python3-cherrypy3 \
python3-git python3-requests python3-redis python3-gnupg \
python3-mysqldb python3-dateutil python3-libnacl python3-openssl
pip3 install timelib==0.2.5
# Install pygit2 package
install_libssh2
install_libgit2
pip3 install pygit2==1.7.0
# Downloading bootstrap-salt.sh script
BOOTSTRAP_VERSION='2021.09.17'
BOOTSTRAP_URL="https://raw.githubusercontent.com/saltstack/salt-bootstrap/v${BOOTSTRAP_VERSION}/bootstrap-salt.sh"
BOOTSTRAP_FILE='bootstrap-salt.sh'
BOOTSTRAP_SHA256='090d652cd6290debce0e3a4eded65086a4272e69446e711eb26f87160593b6a2'
download "${BOOTSTRAP_URL}" "${BOOTSTRAP_FILE}"
check_sha256 "${BOOTSTRAP_FILE}" "${BOOTSTRAP_SHA256}"
# Bootstrap script options:
# https://docs.saltstack.com/en/latest/topics/tutorials/salt_bootstrap.html#command-line-options
## -M: install Salt Master by default
@@ -46,22 +64,17 @@ pip3 install timelib==0.2.5
## -p: Extra-package to install
## -x: Changes the python version used to install a git version of salt
SALT_BOOTSTRAP_OPTS=( -M -N -X -d -P -p salt-api -p salt-call -x "python${PYTHON_VERSION}" )
_WGET_ARGS=()
if [[ "$(uname -i)" == 'armv7l' ]]; then
## -I: allow insecure connections while downloading any files
SALT_BOOTSTRAP_OPTS+=( -I )
_WGET_ARGS+=( --no-check-certificate )
fi
## -I: allow insecure connections while downloading any files
is_arm32 && SALT_BOOTSTRAP_OPTS+=( -I )
echo "Installing saltstack ..."
echo "Option: ${SALT_BOOTSTRAP_OPTS[@]}"
wget ${_WGET_ARGS[@]} -O bootstrap-salt.sh https://bootstrap.saltstack.com
sh bootstrap-salt.sh ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}"
log_info "Installing saltstack ..."
log_debug "Options: ${SALT_BOOTSTRAP_OPTS[@]}"
sh "${BOOTSTRAP_FILE}" ${SALT_BOOTSTRAP_OPTS[@]} git "v${SALT_VERSION}"
chown -R "${SALT_USER}": "${SALT_ROOT_DIR}"
# Configure ssh
echo "Configuring ssh ..."
log_info "Configuring ssh ..."
sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/ssh/ssh_config
{
echo " UserKnownHostsFile /dev/null"
@@ -70,7 +83,7 @@ sed -i -e "s|^[# ]*StrictHostKeyChecking.*$| StrictHostKeyChecking no|" /etc/
} >> /etc/ssh/ssh_config
# Configure logrotate
echo "Configuring logrotate ..."
log_info "Configuring logrotate ..."
# move supervisord.log file to ${SALT_LOGS_DIR}/supervisor/
sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;|" /etc/supervisor/supervisord.conf
@@ -79,7 +92,7 @@ sed -i "s|^[#]*logfile=.*|logfile=${SALT_LOGS_DIR}/supervisor/supervisord.log ;|
sed -i "s|^su root syslog$|su root root|" /etc/logrotate.conf
# Configure supervisor
echo "Configuring supervisor ..."
log_info "Configuring supervisor ..."
# configure supervisord to start salt-master
cat > /etc/supervisor/conf.d/salt-master.conf <<EOF
@@ -110,6 +123,8 @@ stderr_logfile=${SALT_LOGS_DIR}/supervisor/%(program_name)s.log
EOF
# Purge build dependencies and cleanup apt
DEBIAN_FRONTEND=noninteractive apt-get purge -y --auto-remove "${BUILD_DEPENDENCIES[@]}"
DEBIAN_FRONTEND=noninteractive apt-get clean --yes
apt-get purge -y --auto-remove "${BUILD_DEPENDENCIES[@]}"
apt-get clean --yes
rm -rf /var/lib/apt/lists/*
export -n DEBIAN_FRONTEND

4
tests/gitfs/README.md Normal file
View File

@@ -0,0 +1,4 @@
# GitFS Tests
Checks:
- Clone a repository using a RSA 4096 ssh key generated with: `ssh-keygen -t rsa -b 4096 -C "salt@cdalvaro.test" -f gitfs_ssh`

View File

@@ -0,0 +1,2 @@
fileserver_backend:
- gitfs

View File

@@ -0,0 +1,19 @@
gitfs_provider: pygit2
gitfs_base: main
gitfs_privkey: /home/salt/data/keys/gitfs/gitfs_ssh
gitfs_pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub
gitfs_remotes:
- git@github.com:cdalvaro/docker-salt-master-tests.git:
- root: tests/gitfs
git_pillar_provider: pygit2
git_pillar_base: main
git_pillar_branch: main
ext_pillar:
- git:
- git@github.com:cdalvaro/docker-salt-master-tests.git:
- privkey: /home/salt/data/keys/gitfs/gitfs_ssh
- pubkey: /home/salt/data/keys/gitfs/gitfs_ssh.pub
- root: tests/gitfs/pillar

41
tests/gitfs/test.sh Executable file
View File

@@ -0,0 +1,41 @@
#!/usr/bin/env bash
set -e
[ "${DEBUG}" == true ] && set -vx
echo "🧪 Running gitfs tests ..."
IMAGE_NAME=${IMAGE_NAME:-cdalvaro/docker-salt-master}
CONTAINER_NAME=salt_master
PLATFORM=${PLATFORM:-$(docker version --format='{{.Server.Os}}/{{.Server.Arch}}')}
BOOTUP_WAIT_SECONDS=${BOOTUP_WAIT_SECONDS:-60}
function cleanup {
echo "==> Removing ${CONTAINER_NAME} ..."
docker container rm --force "${CONTAINER_NAME}"
}
# trap cleanup EXIT
# Run test instance
echo "==> Starting docker-salt-master (${PLATFORM}) with RSA 4096 ssh key ..."
docker run --rm --detach --name "${CONTAINER_NAME}" \
--publish 4505:4505 --publish 4506:4506 \
--platform "${PLATFORM}" \
--volume "$(pwd)/tests/gitfs/config":/home/salt/data/config:ro \
--volume "$(pwd)/tests/gitfs/data/keys":/home/salt/data/keys \
"${IMAGE_NAME}" || ( echo "container started ❌"; exit 1 )
echo "container started ✅"
# Wait for salt-master bootup
echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..."
sleep "${BOOTUP_WAIT_SECONDS}"
# Check pillars
echo "==> Checking gitfs files ..."
# docker exec "${CONTAINER_NAME}" salt-run cache.clear_git_lock gitfs type=update
# docker exec "${CONTAINER_NAME}" salt-run fileserver.update
FILE_LIST=$(docker exec "${CONTAINER_NAME}" salt-run fileserver.file_list)
echo "${FILE_LIST}"
[[ "${FILE_LIST}" == *test.txt* ]] || ( echo "gitfs files ❌"; exit 1 )
echo "gitfs files ✅"

View File

@@ -52,24 +52,24 @@ echo "container started ✅"
echo "==> Waiting ${BOOTUP_WAIT_SECONDS} seconds for the container to be ready ..."
sleep "${BOOTUP_WAIT_SECONDS}"
# Test salt-api authentication
echo "==> Getting salt-api token ..."
# Test salt-api authentication
echo "==> Getting salt-api token ..."
SALTAPI_TOKEN=$(curl -sSk "${SALTAPI_URL%/}/login" \
-H "Accept: application/x-yaml" \
-d username="${SALTAPI_USER}" \
-d password="${SALTAPI_PASS}" \
-d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4)
-H "Accept: application/x-yaml" \
-d username="${SALTAPI_USER}" \
-d password="${SALTAPI_PASS}" \
-d eauth="${SALTAPI_EAUTH}" | grep 'token:' | cut -d' ' -f 4)
[ -n "${SALTAPI_TOKEN}" ] || ( echo "salt-api token ❌"; exit 1 )
echo "salt-api token ✅"
# Test salt-api command
echo "==> Testing curl command ..."
# Test salt-api command
echo "==> Testing curl command ..."
curl -sSk "${SALTAPI_URL}" \
-H "Accept: application/x-yaml" \
-H "X-Auth-Token: ${SALTAPI_TOKEN}" \
-d client=runner \
-d tgt='*' \
-d fun=test.stream \
-H "Accept: application/x-yaml" \
-H "X-Auth-Token: ${SALTAPI_TOKEN}" \
-d client=runner \
-d tgt='*' \
-d fun=test.stream \
| grep -i 'true' || ( echo "curl command ❌"; exit 1 )
echo "curl command ✅"