api-server: Initial PoC

.editorconfig

@@ -5,6 +5,7 @@ end_of_line = lf
 insert_final_newline = true
 indent_style = space
 charset = utf-8
+indent_size = 4
 
 [*.{yaml,yml}]
 indent_size = 2

.gitignore

@@ -1 +1,52 @@
 /.vagrant/
+/docker-compose.override.yaml
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Environments
+.virtualenv/

api/Dockerfile

@@ -0,0 +1,29 @@
+FROM python:3.11-alpine
+
+RUN apk --no-cache add ansible openssh-client tini su-exec socat
+
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    api-server
+
+USER api-server
+WORKDIR /opt/api-server
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt 
+
+COPY . .
+
+ENV API_PROJECT_DIR=/var/project \
+    API_GALAXY_REQUIREMENTS=/var/project/galaxy-requirements.yml \
+    API_PIP_REQUIREMENTS=/var/project/requirements.txt \
+    API_RUNAS=api-server \
+    PATH="/home/api-server/.local/bin:${PATH}"
+
+VOLUME [ "${API_PROJECT_DIR}" ]
+WORKDIR ${API_PROJECT_DIR}
+
+# switch to root as it must be available
+USER root
+ENTRYPOINT [ "tini", "--", "/opt/api-server/bin/docker-entrypoint.sh" ]

api/api.py

@@ -0,0 +1,53 @@
+import asyncio
+from os import system
+from time import sleep
+from typing import Any, Dict
+from fastapi import FastAPI
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+import ansible_runner
+
+class DeployArgs(BaseModel):
+    extra_vars: Dict[str, Any]
+
+app = FastAPI()
+
+@app.post("/deploy/{service}")
+async def deploy(service: str, args: DeployArgs):
+    finished = False
+    lines = []
+
+    def finish_callback(_):
+        nonlocal finished
+        finished = True
+
+    def event_callback(data: Dict):
+        if 'stdout' in data:
+            lines.append(data['stdout'])
+
+    async def logs():
+        nonlocal lines
+        while not finished:
+            for line in lines:
+                yield line.rstrip() + "\n"
+            lines = []
+            await asyncio.sleep(0.1)
+
+    ansible_runner.run_async(
+        playbook='deploy.yaml', 
+        extravars={
+            'services': [service],
+            **args.extra_vars
+        },
+        private_data_dir='/home/api-server',
+        project_dir='/var/project',
+        inventory='inventory/m2.ini',
+        event_handler=event_callback,
+        finished_callback=finish_callback,
+        settings={
+            'suppress_ansible_output': True
+        }
+    )
+
+    return StreamingResponse(logs(), media_type='text/plain')
+

api/bin/docker-entrypoint.d/00-check-project-mounted.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ -z "$(ls -A ${API_PROJECT_DIR})" ]; then
+    echo "No files found in project dir, maybe you forgot to mount it?" >&2
+    exit 1
+fi

api/bin/docker-entrypoint.d/01-proxy-ssh-agent.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+PROXIED_AUTH_SOCK="${PROXIED_AUTH_SOCK:-/var/run/proxied-ssh-auth.sock}"
+
+if [ -S "${PROXIED_AUTH_SOCK}" ]; then
+    echo "Found previously not closed ssh auth socket, closing."
+    rm ${PROXIED_AUTH_SOCK}
+fi
+
+# This should be run only when run as is specified and SSH Agent socket is present
+if [ -n "$API_RUNAS" ] && [ -S "$SSH_AUTH_SOCK" ]; then
+    echo "Proxying ${SSH_AUTH_SOCK} -> ${PROXIED_AUTH_SOCK} for ${API_RUNAS%%:*}"
+    socat UNIX-LISTEN:${PROXIED_AUTH_SOCK},fork,user=${API_RUNAS%%:*},mode=600 \
+          UNIX-CONNECT:${SSH_AUTH_SOCK} &
+    export SSH_AUTH_SOCK=/var/run/proxied-ssh-auth.sock
+fi 

api/bin/docker-entrypoint.d/10-install-galaxy-requirements.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ -f "${API_GALAXY_REQUIREMENTS}" ]; then
+    echo "Installing galaxy stuff from ${API_GALAXY_REQUIREMENTS}"
+    run-user ansible-galaxy install -r ${API_GALAXY_REQUIREMENTS}
+fi

api/bin/docker-entrypoint.d/20-create-ansible-password-file.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ -n "$ANSIBLE_VAULT_PASSWORD" ]; then
+    export ANSIBLE_VAULT_PASSWORD_FILE=/var/run/secrets/vault-password
+    echo "$ANSIBLE_VAULT_PASSWORD" > $ANSIBLE_VAULT_PASSWORD_FILE
+    unset ANSIBLE_VAULT_PASSWORD
+fi 

api/bin/docker-entrypoint.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# fail on first error
+set -e
+
+function runuser {
+    EXEC=0
+
+    while true; do
+        case "$1" in
+        -E|--exec)
+            EXEC=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            break
+            ;;
+        esac
+    done
+
+    if [ -n "$API_RUNAS" ]; then
+        set -- su-exec "$API_RUNAS" "$@"
+    fi
+    
+    if [ $EXEC -eq 1 ]; then 
+        exec "$@"
+    else
+        "$@"
+    fi
+}
+
+alias run-user=runuser
+
+for part in $(dirname $0)/docker-entrypoint.d/*.sh; do 
+    [ -x $part ] && source $part
+done 
+
+if [ "${1#-}" != "$1" ] || [ $# -eq 0 ]; then
+    set -- uvicorn --app-dir /opt/api-server api:app --host ${API_HOST:-0.0.0.0} --port ${API_PORT:-8080} "$@"
+fi
+
+run-user --exec -- "$@"

api/requirements.txt

@@ -0,0 +1,4 @@
+fastapi>=0.68.0,<0.69.0
+pydantic>=1.8.0,<2.0.0
+uvicorn>=0.15.0,<0.16.0
+ansible-runner>=2.3.0,<2.4.0

docker-compose.yaml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  api:
+    build: api
+    image: registry.kadet.net/mgmt/api:${API_VERSION:-latest}
+    environment:
+      - SSH_AUTH_SOCK
+    ports: 
+      - "8080:8080"
+    command: ['--reload', '--reload-dir', '/opt/api-server']
+    volumes:
+      - .:/var/project
+      - ./api:/opt/api-server
+      - ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}

galaxy-requirements.yml

@@ -0,0 +1,13 @@
+---
+roles:
+  - name: geerlingguy.docker
+    version: 3.0.0
+  - name: geerlingguy.pip
+    version: 2.0.0
+  - name: geerlingguy.mysql
+    version: 3.3.0
+collections:
+  - name: ansible.posix
+    version: 1.4.0
+  - name: community.docker
+    version: 2.7.1

ping.yaml

@@ -0,0 +1,4 @@
+- hosts:
+    - all
+  tasks:
+    - action: ping

services/system/stack.yml

@@ -26,3 +26,16 @@ services:
         - "swarm.cronjob.schedule=0 0 0 * * sun"
       restart_policy:
         condition: none
+
+  volume-cleanup-job:
+    image: docker
+    command: ["docker", "volume", "prune", "-f"]
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    deploy:
+      mode: global
+      labels:
+        - "swarm.cronjob.enable=true"
+        - "swarm.cronjob.schedule=0 0 0 * * *"
+      restart_policy:
+        condition: none

vars/services.yml

@@ -142,6 +142,15 @@ service_config:
           6361376166366438640a323166363063373033356466633839316433613566643734633930363766
           30633736373161383238303262393635393436393637323639366135323530316666623030343633
           6131663563663936316632373565363566343364613666363366
+      - name: github-actions
+        password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          30643365613763383464393263636165373331636139626137376231646536336339613861376631
+          3538383539346566626330326365666164313531336132300a616434623133396665373565353130
+          33366163633136653666343363653464333136626262396337376563623839316536666161373230
+          6562323935356463620a626664313863383730656137383833313766656461386337646531643864
+          61353763393838326561366330653562343133363534656335326332643632643065663437316139
+          6435376534383463346639656261383632323639373930333961
     registry_storage:
       s3:
         accesskey: !vault |
@@ -161,7 +170,7 @@ service_config:
         region: eu-central-003
         regionendpoint: https://s3.eu-central-003.backblazeb2.com
         bucket: kadet-docker
-  portainer: ~
+  portainer: {}
-  system: ~
+  system: {}
 www_data_users:
   - vagrant