.editorconfig

@@ -5,7 +5,6 @@ end_of_line = lf
 insert_final_newline = true
 indent_style = space
 charset = utf-8
-indent_size = 4
 
 [*.{yaml,yml}]
-indent_size = 2
+indent_size = 2

.gitignore

@@ -1,52 +1 @@
-/.vagrant/
+/.vagrant/
-/docker-compose.override.yaml
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Environments
-.virtualenv/

.vscode/settings.json

@@ -6,8 +6,5 @@
         ],
         "https://raw.githubusercontent.com/ansible-community/schemas/main/f/ansible-tasks.json": "tasks/deploy.yml"
     },
-    "yaml.customTags": [
+    "yaml.customTags": ["!vault scalar"]
-        "!vault scalar"
-    ],
-    "python.formatting.provider": "black"
 }

api/Dockerfile

@@ -1,34 +0,0 @@
-FROM python:3.11-alpine
-
-RUN apk --no-cache add ansible openssh-client tini su-exec socat
-
-RUN adduser \
-    --disabled-password \
-    --gecos "" \
-    api-server
-
-RUN mkdir -p /var/run/ansible
-
-USER api-server
-WORKDIR /opt/api-server
-
-COPY requirements.txt .
-RUN pip install -r requirements.txt 
-
-COPY . .
-
-ENV API_PROJECT_DIR=/var/project \
-    API_GALAXY_REQUIREMENTS=/var/project/galaxy-requirements.yml \
-    API_PIP_REQUIREMENTS=/var/project/requirements.txt \
-    API_RUNAS=api-server \
-    PATH="/home/api-server/.local/bin:${PATH}"
-
-
-VOLUME [ "${API_PROJECT_DIR}" ]
-VOLUME [ "/var/run/ansible" ]
-
-WORKDIR ${API_PROJECT_DIR}
-
-# switch to root as it must be available
-USER root
-ENTRYPOINT [ "tini", "--", "/opt/api-server/bin/docker-entrypoint.sh" ]

api/api.py

@@ -1,191 +0,0 @@
-import asyncio
-import uuid
-import ansible_runner
-import os
-import sys
-from typing import Any, Dict
-from fastapi import FastAPI, HTTPException, Request, Security, Depends
-from fastapi.responses import StreamingResponse
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from pydantic import BaseModel, Field
-from ansible_runner import Runner
-from dataclasses import dataclass
-from yaml import safe_load
-
-from starlette.status import HTTP_403_FORBIDDEN
-
-bearer_auth = HTTPBearer()
-
-
-class User(BaseModel):
-    token: str
-    allowed_services: list[str]
-
-
-def load_users(filename) -> dict[str, User]:
-    try:
-        with open(filename, "r") as file:
-            data = safe_load(file)
-
-            if not isinstance(data, list):
-                print(f"{filename} must be list of users", file=sys.stderr)
-                sys.exit(1)
-
-            return {user["token"]: User(**user) for user in data}
-    except FileNotFoundError as e:
-        print(
-            f"File {filename} was not found, please make sure that you provide users file.",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-    except ValueError as e:
-        print(e, file=sys.stderr)
-        sys.exit(1)
-
-
-users = load_users("/etc/api-server/users.yaml")
-
-
-def get_user(bearer_auth: HTTPAuthorizationCredentials = Security(bearer_auth)) -> User:
-    if bearer_auth and bearer_auth.credentials in users:
-        return users[bearer_auth.credentials]
-
-    raise HTTPException(
-        status_code=HTTP_403_FORBIDDEN,
-        detail="Could not validate credentials",
-    )
-
-
-class Link(BaseModel):
-    href: str
-
-
-class DeployArgs(BaseModel):
-    service: str
-    inventory: str | None = None
-    vars: Dict[str, Any] = {}
-
-
-class DeploymentLinks(BaseModel):
-    logs: Link
-    self: Link
-
-
-@dataclass
-class Deployment:
-    id: str
-    args: DeployArgs
-    runner: Runner | None = None
-
-
-class DeploymentDTO(BaseModel):
-    id: str
-    status: str
-    links: DeploymentLinks = Field(..., alias="_links")
-
-    def from_deployment(deployment: Deployment, request: Request):
-        return DeploymentDTO(
-            id=deployment.id,
-            status=deployment.runner.status,
-            _links={
-                "self": {"href": request.url_for("deployment", id=deployment.id)},
-                "logs": {"href": request.url_for("deployment_logs", id=deployment.id)},
-            },
-        )
-
-
-deployments: Dict[str, Deployment] = {}
-
-
-def get_deployment(id: str) -> Deployment:
-    if id not in deployments:
-        raise HTTPException(status_code=404, detail="Deployment was not found")
-
-    return deployments[id]
-
-
-app = FastAPI(
-    docs_url="/",
-    openapi_url="/swagger.json",
-    description="API for services management",
-    title="Server Management API",
-)
-
-
-@app.get("/deployment")
-async def deployment_list(request: Request, user: User = Depends(get_user)):
-    return [
-        DeploymentDTO.from_deployment(deployment, request)
-        for deployment in deployments.values()
-        if deployment.args.service in user.allowed_services
-    ]
-
-
-@app.get("/deployment/{id}/logs")
-async def deployment_logs(
-    deployment: Deployment = Depends(get_deployment), user: User = Depends(get_user)
-):
-    if deployment.args.service not in user.allowed_services:
-        raise HTTPException(
-            status_code=HTTP_403_FORBIDDEN,
-            detail=f"This token does not allow to access {deployment.args.service} deployments.",
-        )
-
-    runner = deployment.runner
-
-    async def stream():
-        stdout = runner.stdout
-
-        while True:
-            while line := stdout.readline():
-                if line == "":
-                    break
-                yield line
-
-            if runner.status != "running":
-                break
-
-            await asyncio.sleep(0.1)
-
-    return StreamingResponse(stream(), media_type="text/plain")
-
-
-@app.get("/deployment/{id}")
-async def deployment(
-    request: Request,
-    deployment: Deployment = Depends(get_deployment),
-    user: User = Depends(get_user),
-):
-    if deployment.args.service not in user.allowed_services:
-        raise HTTPException(
-            status_code=HTTP_403_FORBIDDEN,
-            detail=f"This token does not allow to access {deployment.args.service} deployments.",
-        )
-
-    return DeploymentDTO.from_deployment(deployments[id], request)
-
-
-@app.post("/deployment")
-async def deploy(args: DeployArgs, request: Request, user: User = Depends(get_user)):
-    if args.service not in user.allowed_services:
-        raise HTTPException(
-            status_code=HTTP_403_FORBIDDEN,
-            detail=f"This token does not allow to deploy {args.service} service.",
-        )
-
-    ident = str(uuid.uuid4())
-
-    _, runner = ansible_runner.run_async(
-        playbook="deploy.yaml",
-        ident=ident,
-        extravars={"services": [args.service], **args.vars},
-        private_data_dir="/var/run/ansible",
-        project_dir=os.environ.get("API_PROJECT_DIR", "/var/project"),
-        inventory=args.inventory or os.environ.get("API_INVENTORY"),
-        settings={"suppress_ansible_output": True},
-    )
-
-    deployment = Deployment(id=ident, runner=runner, args=args)
-    deployments[ident] = deployment
-
-    return DeploymentDTO.from_deployment(deployment, request)

api/bin/docker-entrypoint.d/00-check-project-mounted.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-if [ -z "$(ls -A ${API_PROJECT_DIR})" ]; then
-    echo "No files found in project dir, maybe you forgot to mount it?" >&2
-    exit 1
-fi

api/bin/docker-entrypoint.d/01-proxy-ssh-agent.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-PROXIED_AUTH_SOCK="${PROXIED_AUTH_SOCK:-/var/run/proxied-ssh-auth.sock}"
-
-if [ -S "${PROXIED_AUTH_SOCK}" ]; then
-    echo "Found previously not closed ssh auth socket, closing."
-    rm ${PROXIED_AUTH_SOCK}
-fi
-
-# This should be run only when run as is specified and SSH Agent socket is present
-if [ -n "$API_RUNAS" ] && [ -S "$SSH_AUTH_SOCK" ]; then
-    echo "Proxying ${SSH_AUTH_SOCK} -> ${PROXIED_AUTH_SOCK} for ${API_RUNAS%%:*}"
-    socat UNIX-LISTEN:${PROXIED_AUTH_SOCK},fork,user=${API_RUNAS%%:*},mode=600 \
-          UNIX-CONNECT:${SSH_AUTH_SOCK} &
-    export SSH_AUTH_SOCK=/var/run/proxied-ssh-auth.sock
-fi 

api/bin/docker-entrypoint.d/02-make-private-directory-writable.sh

@@ -1 +0,0 @@
-chown $API_RUNAS /var/run/ansible

api/bin/docker-entrypoint.d/10-install-galaxy-requirements.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-if [ -f "${API_GALAXY_REQUIREMENTS}" ]; then
-    echo "Installing galaxy stuff from ${API_GALAXY_REQUIREMENTS}"
-    run-user ansible-galaxy install -r ${API_GALAXY_REQUIREMENTS}
-fi

api/bin/docker-entrypoint.d/20-create-ansible-password-file.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-if [ -n "$ANSIBLE_VAULT_PASSWORD" ]; then
-    export ANSIBLE_VAULT_PASSWORD_FILE=/var/run/secrets/vault-password
-    echo "$ANSIBLE_VAULT_PASSWORD" > $ANSIBLE_VAULT_PASSWORD_FILE
-    unset ANSIBLE_VAULT_PASSWORD
-fi 

api/bin/docker-entrypoint.sh

@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# fail on first error
-set -e
-
-function runuser {
-    EXEC=0
-
-    while true; do
-        case "$1" in
-        -E|--exec)
-            EXEC=1
-            shift
-            ;;
-        --)
-            shift
-            break
-            ;;
-        *)
-            break
-            ;;
-        esac
-    done
-
-    if [ -n "$API_RUNAS" ]; then
-        set -- su-exec "$API_RUNAS" "$@"
-    fi
-    
-    if [ $EXEC -eq 1 ]; then 
-        exec "$@"
-    else
-        "$@"
-    fi
-}
-
-alias run-user=runuser
-
-for part in $(dirname $0)/docker-entrypoint.d/*.sh; do 
-    [ -x $part ] && source $part
-done 
-
-if [ "${1#-}" != "$1" ] || [ $# -eq 0 ]; then
-    set -- uvicorn --app-dir /opt/api-server api:app --host ${API_HOST:-0.0.0.0} --port ${API_PORT:-8080} "$@"
-fi
-
-run-user --exec -- "$@"

api/fixtures/users.yaml

@@ -1,2 +0,0 @@
-- token: test
-  allowed_services: ["wipe-stg"]

api/requirements.txt

@@ -1,4 +0,0 @@
-fastapi>=0.68.0,<0.69.0
-pydantic>=1.8.0,<2.0.0
-uvicorn>=0.15.0,<0.16.0
-ansible-runner>=2.3.0,<2.4.0

docker-compose.yaml

@@ -1,16 +0,0 @@
-version: '3.8'
-
-services:
-  api:
-    build: api
-    image: registry.kadet.net/management/api-server:${API_VERSION:-latest}
-    environment:
-      - SSH_AUTH_SOCK
-    ports: 
-      - "8080:8080"
-    command: ['--reload', '--reload-dir', '/opt/api-server']
-    volumes:
-      - .:/var/project
-      - ./api:/opt/api-server
-      - ./api/fixtures/users.yaml:/etc/api-server/users.yaml:ro
-      - ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}

galaxy-requirements.yml

@@ -1,13 +0,0 @@
----
-roles:
-  - name: geerlingguy.docker
-    version: 3.0.0
-  - name: geerlingguy.pip
-    version: 2.0.0
-  - name: geerlingguy.mysql
-    version: 3.3.0
-collections:
-  - name: ansible.posix
-    version: 1.4.0
-  - name: community.docker
-    version: 2.7.1

main.yaml

@@ -2,8 +2,27 @@
 - hosts: all
   become: yes
 
-- name: Setup server
+  vars_files:
-  import_playbook: setup.yaml
+    - vars/services.yml
+    - vars/environment.yml
+    - vars/databases.yml
+
+  roles:
+    - geerlingguy.docker
+    - geerlingguy.pip
+    - geerlingguy.mysql
+    - kadet.docker-swarm
+
+  tasks:
+    - name: "Add '{{ ansible_user }}' to docker group"
+      user:
+        user: "{{ ansible_user }}"
+        groups: docker
+        append: yes
+    - name: "Add acl package"
+      apt: 
+        name: acl
+        state: present
 
 - name: Deploy services
   import_playbook: deploy.yaml

ping.yaml

@@ -1,4 +0,0 @@
-- hosts:
-    - all
-  tasks:
-    - action: ping

services/management/stack.yml

@@ -1,56 +0,0 @@
-version: "{{ compose_version }}"
-
-services:
-  api:
-    image: registry.kadet.net/management/api-server:{{ api_server_version }}
-    command: ['--proxy-headers']
-    extra_hosts:
-      - manager.swarm.local:{{ swarm_host_address }}
-    networks:
-      - default
-      - "{{ ingress_network }}"
-    environment:
-      - API_INVENTORY=inventory/swarm.ini
-      - ANSIBLE_VAULT_PASSWORD_FILE=/var/run/secrets/vault-password
-    volumes:
-      - ./project:/var/project
-      - private-dir:/var/run/ansible
-    secrets:
-      - source: id-rsa
-        target: /home/api-server/.ssh/id_rsa
-      - source: users_{{ users_config.checksum }}
-        target: /etc/api-server/users.yaml
-      - source: vault-password_{{ vault_password.checksum[:12] }}
-        target: /var/run/secrets/vault-password
-    configs:
-      - source: id-rsa-pub
-        target: /home/api-server/.ssh/id_rsa.pub
-      - source: inventory_{{ inventory_config.checksum }}
-        target: /var/project/inventory/swarm.ini
-    deploy:
-      labels:
-        - traefik.enable=true
-        - traefik.http.routers.{{ service }}.rule=Host(`mgmt.{{ main_domain }}`)
-        - traefik.http.routers.{{ service }}.tls=true
-        - traefik.http.routers.{{ service }}.tls.certresolver=lets-encrypt
-        - traefik.http.services.{{ service }}.loadbalancer.server.port=8080
-      placement:
-        constraints:
-          - node.role == manager
-
-volumes:
-  private-dir: ~
-
-configs:
-  id-rsa-pub:
-    file: ./ssh/id_rsa.pub
-  inventory_{{ inventory_config.checksum }}:
-    file: ./config/inventory.ini
-
-secrets:
-  id-rsa:
-    file: ./ssh/id_rsa
-  users_{{ users_config.checksum }}:
-    file: ./config/users.yaml
-  vault-password_{{ vault_password.checksum[:12] }}:
-    file: ./config/vault-password

services/management/tasks/config.yml

@@ -1,45 +0,0 @@
----
-- name: 'Ensure config directory exists for "{{ service }}"'
-  file:
-    path: "{{ remote_service_path }}/config"
-    state: directory
-    owner: "{{ ansible_user }}"
-  tags:
-    - config
-
-- name: 'Generate inventory file for "{{ service }}"'
-  template:
-    src: "{{ service_path }}/templates/inventory.ini.j2"
-    dest: "{{ remote_service_path }}/config/inventory.ini"
-  register: inventory_config
-  tags:
-    - config
-
-- name: 'Generate users file for "{{ service }}"'
-  template:
-    src: "{{ service_path }}/templates/users.yml.j2"
-    dest: "{{ remote_service_path }}/config/users.yaml"
-  register: users_config
-  tags:
-    - config
-
-- name: 'Generate vault password file for "{{ service }}"'
-  when: api_server_vault_password is defined
-  copy:
-    dest: "{{ remote_service_path }}/config/vault-password"
-    content: "{{ api_server_vault_password }}"
-  register: vault_password
-  tags:
-    - config
-
-- name: 'Ensure SSH config directory exists for "{{ service }}'
-  file:
-    path: "{{ remote_service_path }}/ssh"
-    state: directory
-    owner: "{{ ansible_user }}"
-  tags:
-    - config
-
-- name: 'Generate SSH key pair for "{{ service }}"'
-  community.crypto.openssh_keypair:
-    path: "{{ remote_service_path }}/ssh/id_rsa"

services/management/templates/inventory.ini.j2

@@ -1,7 +0,0 @@
-[main]
-manager.swarm.local
-
-[main:vars]
-ansible_user={{ ansible_user }}
-main_domain={{ main_domain }}
-swarm_addr={{ swarm_addr }}

services/management/templates/users.yml.j2

@@ -1 +0,0 @@
-{{ api_server_users|to_nice_yaml(indent=2) }}

services/management/vars/main.yml

@@ -1,3 +0,0 @@
----
-api_server_version: latest
-api_server_users: []

services/system/stack.yml

@@ -26,16 +26,3 @@ services:
         - "swarm.cronjob.schedule=0 0 0 * * sun"
       restart_policy:
         condition: none
-
-  volume-cleanup-job:
-    image: docker
-    command: ["docker", "volume", "prune", "-f"]
-    volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock"
-    deploy:
-      mode: global
-      labels:
-        - "swarm.cronjob.enable=true"
-        - "swarm.cronjob.schedule=0 0 0 * * *"
-      restart_policy:
-        condition: none

setup.yaml

@@ -1,26 +0,0 @@
----
-- hosts: all
-  become: yes
-
-  vars_files:
-    - vars/services.yml
-    - vars/environment.yml
-    - vars/databases.yml
-
-  roles:
-    - geerlingguy.docker
-    - geerlingguy.pip
-    - geerlingguy.mysql
-    - kadet.docker-swarm
-
-  tasks:
-    - name: "Add '{{ ansible_user }}' to docker group"
-      user:
-        user: "{{ ansible_user }}"
-        groups: docker
-        append: yes
-
-    - name: "Add acl package"
-      apt: 
-        name: acl
-        state: present

vars/services.yml

@@ -142,15 +142,6 @@ service_config:
           6361376166366438640a323166363063373033356466633839316433613566643734633930363766
           30633736373161383238303262393635393436393637323639366135323530316666623030343633
           6131663563663936316632373565363566343364613666363366
-      - name: github-actions
-        password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          30643365613763383464393263636165373331636139626137376231646536336339613861376631
-          3538383539346566626330326365666164313531336132300a616434623133396665373565353130
-          33366163633136653666343363653464333136626262396337376563623839316536666161373230
-          6562323935356463620a626664313863383730656137383833313766656461386337646531643864
-          61353763393838326561366330653562343133363534656335326332643632643065663437316139
-          6435376534383463346639656261383632323639373930333961
     registry_storage:
       s3:
         accesskey: !vault |
@@ -170,28 +161,7 @@ service_config:
         region: eu-central-003
         regionendpoint: https://s3.eu-central-003.backblazeb2.com
         bucket: kadet-docker
-  portainer: {}
+  portainer: ~
-  system: {}
+  system: ~
-  management:
-    api_server_vault_password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          38376138373336636630386339653761646333313238386365653365623335383035643231643162
-          3936353164363934633939306462346262666438346262310a313366353766303833373734383032
-          38643039376636633762653838623565376236653061633734626262616530313734366231326632
-          6530373935323033360a613236366135656436306330306162636661313963613466656163316139
-          32333032613461633834396261623166656365666364393761326438366536306237373661303834
-          3339396235393636346666346365306463643430373639613762
-    api_server_users:
-      - token: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          66663262646661623837363865366463323034376436303738353165306464623933393464666433
-          6632356435323463303436333561333030383163646630350a613831303734336234623366313931
-          39623963653466326239376163313033313139653332343738306634623036666464326131633532
-          3730643635373436380a396662643061653664653366356137366538653431373361383236336362
-          34366665636631373165346338343962613266643036386532326461633766633632373932383663
-          32363837663436663131306238616536623637376137393134623565353436663864333666346231
-          38333435346561306163666533373166363739636533383335663435363431643566383930366163
-          31363565343065313631
-        allowed_services: ["wipe-stg"]
 www_data_users:
   - vagrant