From 81561348e43b3d239931ecd1106cf81f4d5ba8ec Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 26 Oct 2023 10:51:41 +0200 Subject: [PATCH 01/25] postgres refactoring and gitea health check --- _sys/traefik.hcl | 1 + apps/gitea/live.hcl | 7 +++++++ apps/postgresql/live.hcl | 39 +++++++++++++++++++-------------------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/_sys/traefik.hcl b/_sys/traefik.hcl index 0c1e330..4c6b99a 100644 --- a/_sys/traefik.hcl +++ b/_sys/traefik.hcl @@ -18,6 +18,7 @@ job "traefik" { port "api" { static = 81 } + } service { diff --git a/apps/gitea/live.hcl b/apps/gitea/live.hcl index 5b24d0a..3ecfaf2 100644 --- a/apps/gitea/live.hcl +++ b/apps/gitea/live.hcl @@ -27,6 +27,13 @@ job "gitea" { "traefik.enable=true", "traefik.http.routers.gitea.rule=Host(`gitea.service.nr5`)", ] + + check { + type = "http" + path = "/user/login" + interval = "120s" + timeout = "5s" + } } volume "gitea-data" { diff --git a/apps/postgresql/live.hcl b/apps/postgresql/live.hcl index a56ef6b..08bcdce 100644 --- a/apps/postgresql/live.hcl +++ b/apps/postgresql/live.hcl @@ -16,14 +16,28 @@ job "postgres" { access_mode = "single-node-writer" attachment_mode = "file-system" } + + network { + mode = "host" + port "postgres"{ + static = 5432 + } + } + service { + name = "postgres" + port = "postgres" + + tags = [ + "traefik.enable=true", + "traefik.tcp.routers.postgres.rule=HostSNI('postgres.service.nr5')", + ] + + } task "postgres" { driver = "podman" config { image = "docker.io/postgres:13" - network_mode = "host" - port_map { - db = 5432 - } + ports = ["postgres"] } volume_mount { volume = "postgres-data" @@ -42,24 +56,9 @@ job "postgres" { resources { cpu = 1000 memory = 1024 - network { - port "db" { - static = 5432 - } - } - } - service { - name = "postgres" - tags = ["postgres for vault"] - port = "db" - check { - name = "alive" - type = "tcp" - interval = "60s" - timeout = "2s" - } } + } restart { attempts = 10 From 6d35107ff34566a0bde734980ff0d7f1b35c5cf3 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 26 Oct 2023 10:53:21 +0200 Subject: [PATCH 02/25] postgres refactoring and gitea health check --- apps/postgresql/live.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/postgresql/live.hcl b/apps/postgresql/live.hcl index 08bcdce..670e5a6 100644 --- a/apps/postgresql/live.hcl +++ b/apps/postgresql/live.hcl @@ -29,7 +29,7 @@ job "postgres" { tags = [ "traefik.enable=true", - "traefik.tcp.routers.postgres.rule=HostSNI('postgres.service.nr5')", + "traefik.tcp.routers.postgres.rule=HostSNI(`postgres.service.nr5`)", ] } From eff9948c8ee76ea709ae77b84e24946681480383 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 26 Oct 2023 10:53:48 +0200 Subject: [PATCH 03/25] a drone runner which creates nomad jobs --- apps/drone/live-runner-podman-nomad.hcl | 72 +++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 apps/drone/live-runner-podman-nomad.hcl diff --git a/apps/drone/live-runner-podman-nomad.hcl b/apps/drone/live-runner-podman-nomad.hcl new file mode 100644 index 0000000..5601a10 --- /dev/null +++ b/apps/drone/live-runner-podman-nomad.hcl @@ -0,0 +1,72 @@ +job "drone-runner" { + datacenters = [ + "nummer5", + ] + type = "service" + + group "apps" { + count = 1 + + network { + mode = "host" + port "http" { + to = 3000 + } + + } + + service { + name = "drone-runner" + port = "http" + } + + volume "drone-runner" { + type = "csi" + source = "drone-runner" + read_only = false + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + restart { + attempts = 5 + delay = "30s" + } + + task "drone-runner" { + driver = "podman" + env { +# Connection parameters + DRONE_RPC_PROTO="http" + DRONE_RPC_HOST="drone.service.nr5" + DRONE_RPC_SECRET="7eb685ed81d0c34bafc5efa7783c20b2" +# Nomad config + DRONE_JOB_DATACENTER="nummer5" + NOMAD_ADDR="http://nomad.service.nr5" +# Runner agent settings + DRONE_RUNNER_CAPACITY="1" + DRONE_RUNNER_MAX_PROCS="3" + DRONE_RUNNER_NAME="drone-podman-runner1" +# Logging + DRONE_DEBUG="true" + DRONE_TRACE="true" + DRONE_RPC_DUMP_HTTP="true" + DRONE_RPC_DUMP_HTTP_BODY="true" + DRONE_TASK_MEMORY="256" + } + config { + image = "cr.wks/drone-runner-nomad-podman:latest" + volumes = [ + "/run/podman/podman.sock:/var/run/podman.sock", + "/etc/containers:/etc/containers" + ] +#network_mode = "slirp4netns" + ports = ["http"] + } + resources { + cpu = 480 + memory = 200 + } + } +} +} \ No newline at end of file From 9a99887a2b0b964024885be8ef0f1c6579706477 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 26 Oct 2023 10:55:11 +0200 Subject: [PATCH 04/25] a drone runner which creates nomad jobs --- apps/postgresql/live.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/postgresql/live.hcl b/apps/postgresql/live.hcl index 670e5a6..b4ca34c 100644 --- a/apps/postgresql/live.hcl +++ b/apps/postgresql/live.hcl @@ -29,7 +29,7 @@ job "postgres" { tags = [ "traefik.enable=true", - "traefik.tcp.routers.postgres.rule=HostSNI(`postgres.service.nr5`)", + "traefik.tcp.routers.postgres.rule=Host(`postgres.service.nr5`)", ] } From 07315bb2f11b09016812045f3160092f744dfaa1 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Mon, 6 Nov 2023 18:12:40 +0100 Subject: [PATCH 05/25] no traefik router --- apps/postgresql/live.hcl | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/apps/postgresql/live.hcl b/apps/postgresql/live.hcl index b4ca34c..0c8998b 100644 --- a/apps/postgresql/live.hcl +++ b/apps/postgresql/live.hcl @@ -1,8 +1,3 @@ -#To Configure vault -# vault secrets enable database -# vault write database/config/postgresql plugin_name=postgresql-database-plugin connection_url="postgresql://{{username}}:{{password}}@postgres.service.consul:5432/postgres?sslmode=disable" allowed_roles="*" username="root" password="rootpassword" -# vault write database/roles/readonly db_name=postgresql creation_statements=@readonly.sql default_ttl=1h max_ttl=24h - job "postgres" { datacenters = ["nummer5"] type = "service" @@ -26,12 +21,10 @@ job "postgres" { service { name = "postgres" port = "postgres" - - tags = [ - "traefik.enable=true", - "traefik.tcp.routers.postgres.rule=Host(`postgres.service.nr5`)", - ] - + #tags = [ + # "traefik.enable=true", + # "traefik.tcp.routers.postgres.rule=Host(`postgres.service.nr5`)", + #] } task "postgres" { driver = "podman" From 00e47aa976b51edfc1c1ec69fe074b9138b749e5 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Tue, 14 Nov 2023 10:16:31 +0100 Subject: [PATCH 06/25] dmarc ui --- apps/dmarc/live.hcl | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 apps/dmarc/live.hcl diff --git a/apps/dmarc/live.hcl b/apps/dmarc/live.hcl new file mode 100644 index 0000000..7b471f8 --- /dev/null +++ b/apps/dmarc/live.hcl @@ -0,0 +1,62 @@ +job "dmarc" { + datacenters = [ + "nummer5", + ] + type = "service" + + group "apps" { + count = 1 + + network { + mode = "host" + port "http" { + to = 8080 + } + } + service { + name = "dmarc" + port = "http" + + tags = [ + "traefik.enable=true", + "traefik.http.routers.dmarc.rule=Host(`dmarc.service.nr5`)", + ] + + } + + restart { + attempts = 5 + delay = "30s" + } + + task "dmarc" { + driver = "podman" + + config { + image = "cr.wks/dmarc-report:latest" + ports = ["http"] + } + + env { + TZ = "Europe/Berlin" + REPORT_DB_TYPE = "pgsql" + PARSER_DB_TYPE = "pgsql" + PARSER_DB_HOST = "postgres.service.nr5" + PARSER_DB_NAME = "dmarc-srg" + PARSER_DB_USER = "dmarc" + PARSER_DB_PASSWORD = "4XSS4gKpheSBoMsIs" + PARSER_IMAP_PORT = "143" + PARSER_IMAP_HOST = "xximap.maketank.net" + PARSER_IMAP_USER = "dmarc-inbox@maketank.net" + PARSER_IMAP_PASSWORD = "j2Kwd6mVPZw2yMLw2gIKwn" + PARSER_IMAP_READ_FOLDER = "Inbox" + } + + resources { + cpu = 200 + memory = 512 + } + + } + } +} \ No newline at end of file From e0c1edccbbf119059ae1dfa2005133a5d2368a07 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Tue, 14 Nov 2023 17:35:13 +0100 Subject: [PATCH 07/25] fixes #10 - proper config for docker-registry-ui --- apps/docker-registry/live-ui.hcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/docker-registry/live-ui.hcl b/apps/docker-registry/live-ui.hcl index 4354496..98d0719 100644 --- a/apps/docker-registry/live-ui.hcl +++ b/apps/docker-registry/live-ui.hcl @@ -39,8 +39,8 @@ job "docker-registry-ui" { env { DELETE_IMAGES = "true" REGISTRY_TITLE = "Nummer5 Reg" - NGINX_PROXY_PASS_URL_DISABLED = "http://cr.wks" - REGISTRY_URL = "http://cr.wks:5000" + NGINX_PROXY_PASS_URL = "http://cr.wks" + XX_REGISTRY_URL = "http://cr.wks:5000" URL = "http://cr-ui.service.nr5" SINGLE_REGISTRY = "true" SHOW_CONTENT_DIGEST = "true" From 1eb2a0e5fd7819e9bceec685e0c63e21e7fc8d70 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Tue, 14 Nov 2023 17:36:57 +0100 Subject: [PATCH 08/25] fixes #9 - there we go --- apps/dmarc/live.hcl | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/apps/dmarc/live.hcl b/apps/dmarc/live.hcl index 7b471f8..fed3ef7 100644 --- a/apps/dmarc/live.hcl +++ b/apps/dmarc/live.hcl @@ -10,7 +10,7 @@ job "dmarc" { network { mode = "host" port "http" { - to = 8080 + to = 80 } } service { @@ -40,21 +40,23 @@ job "dmarc" { env { TZ = "Europe/Berlin" REPORT_DB_TYPE = "pgsql" - PARSER_DB_TYPE = "pgsql" - PARSER_DB_HOST = "postgres.service.nr5" - PARSER_DB_NAME = "dmarc-srg" - PARSER_DB_USER = "dmarc" - PARSER_DB_PASSWORD = "4XSS4gKpheSBoMsIs" + REPORT_DB_HOST = "postgres.service.nr5" + REPORT_DB_PORT = "5432" + REPORT_DB_NAME = "dmarc-srg" + REPORT_DB_USER = "dmarc" + REPORT_DB_PASS = "4XSS4gKpheSBoMsIs" PARSER_IMAP_PORT = "143" - PARSER_IMAP_HOST = "xximap.maketank.net" + PARSER_IMAP_SERVER = "116.202.109.243" #"imap.maketank.net" PARSER_IMAP_USER = "dmarc-inbox@maketank.net" - PARSER_IMAP_PASSWORD = "j2Kwd6mVPZw2yMLw2gIKwn" + PARSER_IMAP_PASS = "j2Kwd6mVPZw2yMLw2gIKwn" PARSER_IMAP_READ_FOLDER = "Inbox" + PARSER_IMAP_MOVE_FOLDER = "Processed" + PARSER_IMAP_MOVE_FOLDER_ERR = "Error" } resources { - cpu = 200 - memory = 512 + cpu = 100 + memory = 128 } } From 2b409943371cde24a5acb5f79d4ae6414a4b7992 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Tue, 14 Nov 2023 18:26:13 +0100 Subject: [PATCH 09/25] homer. a frontpage --- apps/homer/live.hcl | 50 +++++++++++++++++++++++++++++++++++++++++++ apps/homer/volume.hcl | 20 +++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 apps/homer/live.hcl create mode 100644 apps/homer/volume.hcl diff --git a/apps/homer/live.hcl b/apps/homer/live.hcl new file mode 100644 index 0000000..49e82d6 --- /dev/null +++ b/apps/homer/live.hcl @@ -0,0 +1,50 @@ +job "homer" { + datacenters = ["nummer5"] + + group "apps" { + count = 1 + + network { + port "http"{ + to = 8080 + } + } + + service { + name = "homer" + port = "http" + + tags = [ + "traefik.enable=true", + "traefik.http.routers.http.rule=Host(`homer.service.nr5`)", + ] + + } + + volume "homer-assets" { + type = "csi" + read_only = false + source = "homer-assets" + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + task "homer" { + driver = "podman" + + config { + image = "b4bz/homer:latest" + ports = ["http"] + } + + volume_mount { + volume = "homer-assets" + destination = "/www/assets" + } + } + resources { + cpu = 10 + memory = 32 + } + } +} diff --git a/apps/homer/volume.hcl b/apps/homer/volume.hcl new file mode 100644 index 0000000..2cd7ba5 --- /dev/null +++ b/apps/homer/volume.hcl @@ -0,0 +1,20 @@ +type = "csi" +id = "homer-assets" +name = "homer-assets" +plugin_id = "nfs" + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +context { + server = "ebin01.wks" + share = "/data/raid1-ssd/app-data/homer-assets" + mountPermissions = "0" +} + +mount_options { + fs_type = "nfs" + mount_flags = [ "timeo=30", "vers=3", "_netdev" , "nolock" ] +} \ No newline at end of file From 3f600574088119edc1fde054a41e5d31bc3ba265 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 15 Nov 2023 13:15:03 +0100 Subject: [PATCH 10/25] building now --- apps/homer/live.hcl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/homer/live.hcl b/apps/homer/live.hcl index 49e82d6..f0f59bd 100644 --- a/apps/homer/live.hcl +++ b/apps/homer/live.hcl @@ -16,7 +16,7 @@ job "homer" { tags = [ "traefik.enable=true", - "traefik.http.routers.http.rule=Host(`homer.service.nr5`)", + "traefik.http.routers.homer.rule=Host(`homer.service.nr5`)", ] } @@ -41,10 +41,11 @@ job "homer" { volume = "homer-assets" destination = "/www/assets" } + + resources { + cpu = 10 + memory = 32 + } } - resources { - cpu = 10 - memory = 32 - } } } From 40b85e58cbe5291b5e3e2a99670638640b523b45 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 16 Nov 2023 12:06:30 +0100 Subject: [PATCH 11/25] acl allow for now --- _sys/etc_consul.d/acl.hcl | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 _sys/etc_consul.d/acl.hcl diff --git a/_sys/etc_consul.d/acl.hcl b/_sys/etc_consul.d/acl.hcl new file mode 100644 index 0000000..2b4bf85 --- /dev/null +++ b/_sys/etc_consul.d/acl.hcl @@ -0,0 +1,5 @@ +acl = { + enabled = true + default_policy = "allow" + enable_token_persistence = true +} \ No newline at end of file From aa3961ba31638b613e4e09b9373dd358f307fa0e Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 16 Nov 2023 12:06:57 +0100 Subject: [PATCH 12/25] initial vault config --- _sys/vault/vault-service-policy.hcl | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 _sys/vault/vault-service-policy.hcl diff --git a/_sys/vault/vault-service-policy.hcl b/_sys/vault/vault-service-policy.hcl new file mode 100644 index 0000000..302fe60 --- /dev/null +++ b/_sys/vault/vault-service-policy.hcl @@ -0,0 +1,13 @@ +https://developer.hashicorp.com/vault/tutorials/day-one-consul/deployment-guide +#consul acl policy create -name vault-service -rules @vault-service-policy.hcl +#consul acl token create \ +# -description "Vault Service Token" \ +# -policy-name vault-service + + +service "vault" { policy = "write" } +key_prefix "vault/" { policy = "write" } +agent_prefix "" { policy = "read" } +session_prefix "" { policy = "write" } + + From 63f1bab168716d4db97c4d69fbfd786858921e09 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 16 Nov 2023 12:07:15 +0100 Subject: [PATCH 13/25] podman runner for droneci --- apps/drone/live-runner-podman.hcl | 73 +++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 apps/drone/live-runner-podman.hcl diff --git a/apps/drone/live-runner-podman.hcl b/apps/drone/live-runner-podman.hcl new file mode 100644 index 0000000..8b09181 --- /dev/null +++ b/apps/drone/live-runner-podman.hcl @@ -0,0 +1,73 @@ +job "drone-runner" { + datacenters = [ + "nummer5", + ] + type = "service" + + group "apps" { + count = 1 + + network { + mode = "host" + port "http" { + to = 3000 + } + + } + + service { + name = "drone-runner" + port = "http" + } + + volume "drone-runner" { + type = "csi" + source = "drone-runner" + read_only = false + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + restart { + attempts = 5 + delay = "30s" + } + + task "drone-runner" { + driver = "podman" + volume_mount { + volume = "drone-runner" + destination = "/var/lib/docker" + read_only = false + } + + config { + image = "cr.wks/drone/drone-runner-podman:latest" + force_pull = true + ports = ["http"] + volumes = [ + "/run/podman/podman.sock:/run/podman/podman.sock", + "/run/podman/podman.sock:/var/run/docker.sock", + "/etc/containers:/etc/containers" + ] + + } + + env { + TZ = "Europe/Berlin" + DRONE_RUNNER_NAME = "drone-runner01" + DRONE_RPC_SECRET = "7eb685ed81d0c34bafc5efa7783c20b2" + DRONE_RPC_HOST = "drone.service.nr5" + DRONE_RPC_PROTO = "http" + DRONE_LOGS_DEBUG = true + DRONE_LOGS_TRACE = true + } + + resources { + cpu = 500 + memory = 128 + } + + } + } +} From d9cb34016914aff404ab6ee82200f00c9c653a86 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Thu, 30 Nov 2023 17:18:03 +0100 Subject: [PATCH 14/25] drone runner fron docker.io again --- apps/drone/live-runner-podman.hcl | 2 +- apps/drone/live-runner.hcl | 20 ++++---------------- apps/drone/live.hcl | 7 +++++++ apps/gitea/live.hcl | 3 +++ apps/postgresql/live.hcl | 2 +- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/apps/drone/live-runner-podman.hcl b/apps/drone/live-runner-podman.hcl index 8b09181..4b07917 100644 --- a/apps/drone/live-runner-podman.hcl +++ b/apps/drone/live-runner-podman.hcl @@ -37,7 +37,7 @@ job "drone-runner" { driver = "podman" volume_mount { volume = "drone-runner" - destination = "/var/lib/docker" + destination = "/data" read_only = false } diff --git a/apps/drone/live-runner.hcl b/apps/drone/live-runner.hcl index 79f9a09..90e04b5 100644 --- a/apps/drone/live-runner.hcl +++ b/apps/drone/live-runner.hcl @@ -20,14 +20,6 @@ job "drone-runner" { port = "http" } - volume "drone-runner" { - type = "csi" - source = "drone-runner" - read_only = false - access_mode = "single-node-writer" - attachment_mode = "file-system" - } - restart { attempts = 5 delay = "30s" @@ -35,14 +27,10 @@ job "drone-runner" { task "drone-runner" { driver = "podman" - volume_mount { - volume = "drone-runner" - destination = "/drone" - read_only = false - } config { - image = "docker.io/drone/drone-runner-docker:latest" + image = "drone/drone-runner-docker:latest" + force_pull = true ports = ["http"] volumes = [ "/var/run/podman/podman.sock:/var/run/docker.sock", @@ -62,8 +50,8 @@ job "drone-runner" { } resources { - cpu = 500 - memory = 128 + cpu = 100 + memory = 64 } } diff --git a/apps/drone/live.hcl b/apps/drone/live.hcl index 55585fb..7936369 100644 --- a/apps/drone/live.hcl +++ b/apps/drone/live.hcl @@ -23,6 +23,13 @@ job "drone" { "traefik.enable=true", "traefik.http.routers.drone.rule=Host(`drone.service.nr5`)", ] + + check { + type = "http" + path = "/welcome" + interval = "120s" + timeout = "5s" + } } volume "drone-data" { diff --git a/apps/gitea/live.hcl b/apps/gitea/live.hcl index 3ecfaf2..86c4cc4 100644 --- a/apps/gitea/live.hcl +++ b/apps/gitea/live.hcl @@ -51,6 +51,7 @@ job "gitea" { task "gitea" { driver = "podman" + volume_mount { volume = "gitea-data" destination = "/data" @@ -60,6 +61,7 @@ job "gitea" { config { image = "docker.io/gitea/gitea:latest" ports = ["ssh", "http"] + force_pull = true } env { @@ -80,6 +82,7 @@ job "gitea" { GITEA__packages__ENABLED = "true" GITEA__log__LEVEL = "warn" GITEA__actions__ENABLED = "true" + GITEA__webhook__ALLOWED_HOST_LIST = "private" } diff --git a/apps/postgresql/live.hcl b/apps/postgresql/live.hcl index 0c8998b..4042bb9 100644 --- a/apps/postgresql/live.hcl +++ b/apps/postgresql/live.hcl @@ -48,7 +48,7 @@ job "postgres" { resources { cpu = 1000 - memory = 1024 + memory = 512 } From 1b4e33c103c749effaa3404feb6ebba51058bbb1 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 13 Dec 2023 12:33:31 +0100 Subject: [PATCH 15/25] testing droneci --- .drone.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.drone.yml b/.drone.yml index 72582ac..75d49a3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -5,6 +5,10 @@ name: nomad-nummer5 platform: os: linux arch: arm64 + + +environment: + TARGET_HOST: "test.chaos" steps: - name: test @@ -12,3 +16,4 @@ steps: commands: - echo hello - echo world + - echo $TARGET_HOST From f34291aa05a985a2b12ee453dc0b5a7a1038e892 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 13 Dec 2023 13:53:09 +0100 Subject: [PATCH 16/25] enabled jsonnet in drone --- apps/drone/live.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/drone/live.hcl b/apps/drone/live.hcl index 7936369..2217553 100644 --- a/apps/drone/live.hcl +++ b/apps/drone/live.hcl @@ -67,6 +67,7 @@ job "drone" { DRONE_RPC_SECRET = "7eb685ed81d0c34bafc5efa7783c20b2" DRONE_SERVER_HOST = "drone.service.nr5" DRONE_SERVER_PROTO = "http" + DRONE_JSONNET_ENABLED = true DRONE_LOGS_DEBUG = true DRONE_LOGS_TRACE = true } From 76f6af91877dcd99a6283692f5d64503e34cc1f5 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Fri, 15 Dec 2023 18:26:22 +0100 Subject: [PATCH 17/25] only one capacity --- apps/drone/live-runner.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/drone/live-runner.hcl b/apps/drone/live-runner.hcl index 90e04b5..0a06588 100644 --- a/apps/drone/live-runner.hcl +++ b/apps/drone/live-runner.hcl @@ -45,6 +45,7 @@ job "drone-runner" { DRONE_RPC_SECRET = "7eb685ed81d0c34bafc5efa7783c20b2" DRONE_RPC_HOST = "drone.service.nr5" DRONE_RPC_PROTO = "http" + DRONE_RUNNER_CAPACITY = 1 DRONE_LOGS_DEBUG = true DRONE_LOGS_TRACE = true } From c9c117a24de78c8dc0ce24e16e7785c36518b6ce Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 20 Dec 2023 12:57:06 +0100 Subject: [PATCH 18/25] admin user --- apps/drone/live.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/drone/live.hcl b/apps/drone/live.hcl index 2217553..42f2ac1 100644 --- a/apps/drone/live.hcl +++ b/apps/drone/live.hcl @@ -70,6 +70,7 @@ job "drone" { DRONE_JSONNET_ENABLED = true DRONE_LOGS_DEBUG = true DRONE_LOGS_TRACE = true + DRONE_USER_CREATE = "username:do,admin:true" } resources { From e85dd7e44b7244c726eab07b064b02c118e0cbed Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 20 Dec 2023 12:57:20 +0100 Subject: [PATCH 19/25] always pull --- apps/apt-cacher-ng/live.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/apt-cacher-ng/live.hcl b/apps/apt-cacher-ng/live.hcl index 28e2fc1..43e12a2 100644 --- a/apps/apt-cacher-ng/live.hcl +++ b/apps/apt-cacher-ng/live.hcl @@ -40,6 +40,7 @@ job "apt-cacher-ng" { config { image = "cr.wks/apt-cacher-ng" ports = ["http"] + force_pull = true } volume_mount { From 16d9406d291d06c7577a30a8ad4965757c8fec51 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 17 Jan 2024 11:38:48 +0100 Subject: [PATCH 20/25] mqtt exporter for prometheus --- apps/mosquitto-prometheus-exporter/live.hcl | 38 +++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 apps/mosquitto-prometheus-exporter/live.hcl diff --git a/apps/mosquitto-prometheus-exporter/live.hcl b/apps/mosquitto-prometheus-exporter/live.hcl new file mode 100644 index 0000000..3b12dc4 --- /dev/null +++ b/apps/mosquitto-prometheus-exporter/live.hcl @@ -0,0 +1,38 @@ +job "mosquitto-prometheus-exporter" { + datacenters = ["nummer5"] + + group "apps" { + count = 1 + + network { + port "http"{ + to = 9234 + } + } + + service { + name = "mosquitto-prometheus-exporter" + port = "http" + + tags = [ + "traefik.enable=true", + "traefik.http.routers.mosquitto-pormetheus-exporter.rule=Host(`mosquitto-prometheus-exporter.service.nr5`)", + ] + + } + + task "server" { + driver = "podman" + + config { + image = "cr.wks/mosquitto-prometheus-exporter" + ports = ["http"] + force_pull = true + + args = [ + "--endpoint", "tcp://mqtt.wks:1883" + ] + } + } + } +} From 1c24642405a88b2031b17f06069e4399dbbded85 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 17 Jan 2024 11:38:59 +0100 Subject: [PATCH 21/25] lesser resources for nfs stuff --- _sys/nfs-controller.hcl | 4 ++-- _sys/nfs-nodes.hcl | 4 ++-- apps/drone/live-runner.hcl | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/_sys/nfs-controller.hcl b/_sys/nfs-controller.hcl index df19ecb..a5fe5a2 100644 --- a/_sys/nfs-controller.hcl +++ b/_sys/nfs-controller.hcl @@ -30,8 +30,8 @@ job "plugin-nfs-controller" { } resources { - cpu = 250 - memory = 128 + cpu = 50 + memory = 15 } } } diff --git a/_sys/nfs-nodes.hcl b/_sys/nfs-nodes.hcl index 0723589..9316be5 100644 --- a/_sys/nfs-nodes.hcl +++ b/_sys/nfs-nodes.hcl @@ -34,8 +34,8 @@ job "plugin-nfs-nodes" { } resources { - cpu = 250 - memory = 128 + cpu = 50 + memory = 10 } } } diff --git a/apps/drone/live-runner.hcl b/apps/drone/live-runner.hcl index 0a06588..baa23f8 100644 --- a/apps/drone/live-runner.hcl +++ b/apps/drone/live-runner.hcl @@ -51,8 +51,8 @@ job "drone-runner" { } resources { - cpu = 100 - memory = 64 + cpu = 300 + memory = 1500 } } From 9ec3788dfee35d797ee1f9ce8b6b1fa07bf2537f Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Wed, 17 Jan 2024 18:30:10 +0100 Subject: [PATCH 22/25] resources for prometheus mqtt exporter --- apps/drone/live-runner.hcl | 1 + apps/mosquitto-prometheus-exporter/live.hcl | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/apps/drone/live-runner.hcl b/apps/drone/live-runner.hcl index baa23f8..fe3a0bc 100644 --- a/apps/drone/live-runner.hcl +++ b/apps/drone/live-runner.hcl @@ -48,6 +48,7 @@ job "drone-runner" { DRONE_RUNNER_CAPACITY = 1 DRONE_LOGS_DEBUG = true DRONE_LOGS_TRACE = true + DOCKER_BUILDKIT = 1 } resources { diff --git a/apps/mosquitto-prometheus-exporter/live.hcl b/apps/mosquitto-prometheus-exporter/live.hcl index 3b12dc4..e1eb208 100644 --- a/apps/mosquitto-prometheus-exporter/live.hcl +++ b/apps/mosquitto-prometheus-exporter/live.hcl @@ -33,6 +33,10 @@ job "mosquitto-prometheus-exporter" { "--endpoint", "tcp://mqtt.wks:1883" ] } + resources { + cpu = 50 + memory = 10 + } } } } From 067f779597b9854e2e5b1dcf8015d4b2f964de8d Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Mon, 5 Feb 2024 16:49:26 +0100 Subject: [PATCH 23/25] redis --- apps/redis/live.hcl | 54 +++++++++++++++++++++++++++++++++++++++++++ apps/redis/volume.hcl | 21 +++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 apps/redis/live.hcl create mode 100644 apps/redis/volume.hcl diff --git a/apps/redis/live.hcl b/apps/redis/live.hcl new file mode 100644 index 0000000..580663d --- /dev/null +++ b/apps/redis/live.hcl @@ -0,0 +1,54 @@ + +job "redis" { + datacenters = ["nummer5"] + + group "cache" { + + count = 1 + + volume "volume0" { + type = "csi" + source = "redis" + read_only = false + attachment_mode = "file-system" + access_mode = "single-node-writer" + } + + network { + port "redis" { + static = 6379 + } + } + service { + name = "redis" + port = "redis" + tags = [ + "traefik.enable=true", + "traefik.tcp.routers.redis.rule=HostSNI(`redis.service.nr5`)", + # "traefik.tcp.routers.redis.entryPoints=redis", + # "traefik.tcp.routers.redis.service=redis", + # services (needed for TCP) + "traefik.tcp.services.redis.loadbalancer.server.port=6379", + ] + } + + task "redis" { + driver = "podman" + + config { + image = "redis" + ports = ["redis"] + } + + volume_mount { + volume = "volume0" + destination = "/data" + } + + resources { + cpu = 500 + memory = 256 + } + } + } +} \ No newline at end of file diff --git a/apps/redis/volume.hcl b/apps/redis/volume.hcl new file mode 100644 index 0000000..33d7ce9 --- /dev/null +++ b/apps/redis/volume.hcl @@ -0,0 +1,21 @@ +type = "csi" +id = "redis" +name = "redis" +plugin_id = "nfs" +external_id = "redis" + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +context { + server = "ebin02.wks" + share = "/data/raid1-ssd/app-data/redis-data" + mountPermissions = "0" +} + +mount_options { + fs_type = "nfs" + mount_flags = [ "timeo=30", "vers=3", "_netdev" , "nolock" ] +} \ No newline at end of file From eb01682a570dd26cf148990306cc9e1337966553 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Tue, 6 Feb 2024 13:26:22 +0100 Subject: [PATCH 24/25] netbox --- apps/netbox/live.hcl | 72 ++++++++++++++++++++++++++++++++++++++++++ apps/netbox/volume.hcl | 20 ++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 apps/netbox/live.hcl create mode 100644 apps/netbox/volume.hcl diff --git a/apps/netbox/live.hcl b/apps/netbox/live.hcl new file mode 100644 index 0000000..8b6f73d --- /dev/null +++ b/apps/netbox/live.hcl @@ -0,0 +1,72 @@ +job "netbox" { + datacenters = [ + "nummer5", + ] + type = "service" + + group "apps" { + count = 1 + + network { + port "http" { + to = 8000 + } + } + + service { + name = "netbox" + port = "http" + + tags = [ + "traefik.enable=true", + "traefik.http.routers.netbox.rule=Host(`netbox.service.nr5`)", + ] + } + + volume "netbox" { + type = "csi" + source = "netbox" + read_only = false + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + restart { + attempts = 5 + delay = "60s" + } + + task "netbox" { + driver = "podman" + volume_mount { + volume = "netbox" + destination = "/config" + read_only = false + } + + config { + image = "docker.io/netboxcommunity/netbox" + ports = ["http"] + } + env { + TZ="Europe/Berlin" + SUPERUSER_EMAIL="udo@maketank.net" + SUPERUSER_PASSWORD="superu" + ALLOWED_HOST="netbox.service.nr5" + DB_WAIT_DEBUG=1 + DB_NAME="netbox" + DB_USER="netbox" + DB_PASSWORD="IK$Wb5TGhphNo:-WktT" + DB_HOST="postgres.service.nr5" + DB_PORT="5472" + REDIS_HOST="redis.service.nr5" + REDIS_PORT="6379" + } + resources { + cpu = 400 + memory = 128 + } + + } + } +} \ No newline at end of file diff --git a/apps/netbox/volume.hcl b/apps/netbox/volume.hcl new file mode 100644 index 0000000..64c09ea --- /dev/null +++ b/apps/netbox/volume.hcl @@ -0,0 +1,20 @@ +type = "csi" +id = "netbox" +name = "netbox" +plugin_id = "nfs" + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +context { + server = "ebin02.wks" + share = "/data/raid1-ssd/app-data/netbox" + mountPermissions = "0" +} + +mount_options { + fs_type = "nfs" + mount_flags = [ "timeo=30", "vers=3", "_netdev" , "nolock" ] +} \ No newline at end of file From 91fff0b27607ec94c60e302d59df5441e1df0c42 Mon Sep 17 00:00:00 2001 From: Udo Waechter Date: Mon, 19 Feb 2024 14:49:55 +0100 Subject: [PATCH 25/25] updates and ring86 docks --- README.md | 12 ++++++++++++ _sys/vault/role-ssh-signer.json | 13 +++++++++++++ apps/drone/live-runner.hcl | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 _sys/vault/role-ssh-signer.json diff --git a/README.md b/README.md index 9cd1546..3a165c2 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,15 @@ NFS - https://github.com/thatsk/nfs-csi-nomad/tree/main Podman - https://github.com/hashicorp/nomad-driver-podman + + + + + +# Datacenter: ring86 + +# auto.chaos +podman run -d --replace -e 1883 -p 1883:1883 --mount=type=bind,source=/etc/mosquitto,destination=/mosquitto --tz=Europe/Berlin --name=mosquitto-mqtt cr.wks/mosquitto:latest +podman run -d --replace -e 9234 -p 0.0.0.0:9234:9234 --tz=Europe/Berlin --name=mosquitto-exporter cr.wks/mosquitto-prometheus-exporter --endpoint "tcp://mqtt:1883" + + diff --git a/_sys/vault/role-ssh-signer.json b/_sys/vault/role-ssh-signer.json new file mode 100644 index 0000000..65c6f2e --- /dev/null +++ b/_sys/vault/role-ssh-signer.json @@ -0,0 +1,13 @@ +#https://developer.hashicorp.com/vault/docs/secrets/ssh/signed-ssh-certificates +{ + "algorithm_signer": "rsa-sha2-256", + "allow_user_certificates": true, + "allowed_users": "*", + "allowed_extensions": "permit-pty,permit-port-forwarding", + "default_extensions": { + "permit-pty": "" + }, + "key_type": "ca", + "default_user": "admini", + "ttl": "30m0s" +} diff --git a/apps/drone/live-runner.hcl b/apps/drone/live-runner.hcl index fe3a0bc..39fe2b4 100644 --- a/apps/drone/live-runner.hcl +++ b/apps/drone/live-runner.hcl @@ -29,7 +29,7 @@ job "drone-runner" { driver = "podman" config { - image = "drone/drone-runner-docker:latest" + image = "docker.io/drone/drone-runner-docker:latest" force_pull = true ports = ["http"] volumes = [