management: Initial config for api-server service

.editorconfig

@@ -5,6 +5,7 @@ end_of_line = lf
 insert_final_newline = true
 indent_style = space
 charset = utf-8
+indent_size = 4
 
 [*.{yaml,yml}]
-indent_size = 2
+indent_size = 2

.gitignore

@@ -1 +1,52 @@
-/.vagrant/
+/.vagrant/
+/docker-compose.override.yaml
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Environments
+.virtualenv/

.vscode/settings.json

@@ -6,5 +6,8 @@
         ],
         "https://raw.githubusercontent.com/ansible-community/schemas/main/f/ansible-tasks.json": "tasks/deploy.yml"
     },
-    "yaml.customTags": ["!vault scalar"]
+    "yaml.customTags": [
+        "!vault scalar"
+    ],
+    "python.formatting.provider": "black"
 }

api/Dockerfile

@@ -0,0 +1,34 @@
+FROM python:3.11-alpine
+
+RUN apk --no-cache add ansible openssh-client tini su-exec socat
+
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    api-server
+
+RUN mkdir -p /var/run/ansible
+
+USER api-server
+WORKDIR /opt/api-server
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt 
+
+COPY . .
+
+ENV API_PROJECT_DIR=/var/project \
+    API_GALAXY_REQUIREMENTS=/var/project/galaxy-requirements.yml \
+    API_PIP_REQUIREMENTS=/var/project/requirements.txt \
+    API_RUNAS=api-server \
+    PATH="/home/api-server/.local/bin:${PATH}"
+
+
+VOLUME [ "${API_PROJECT_DIR}" ]
+VOLUME [ "/var/run/ansible" ]
+
+WORKDIR ${API_PROJECT_DIR}
+
+# switch to root as it must be available
+USER root
+ENTRYPOINT [ "tini", "--", "/opt/api-server/bin/docker-entrypoint.sh" ]

api/api.py

@@ -0,0 +1,191 @@
+import asyncio
+import uuid
+import ansible_runner
+import os
+import sys
+from typing import Any, Dict
+from fastapi import FastAPI, HTTPException, Request, Security, Depends
+from fastapi.responses import StreamingResponse
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import BaseModel, Field
+from ansible_runner import Runner
+from dataclasses import dataclass
+from yaml import safe_load
+
+from starlette.status import HTTP_403_FORBIDDEN
+
+bearer_auth = HTTPBearer()
+
+
+class User(BaseModel):
+    token: str
+    allowed_services: list[str]
+
+
+def load_users(filename) -> dict[str, User]:
+    try:
+        with open(filename, "r") as file:
+            data = safe_load(file)
+
+            if not isinstance(data, list):
+                print(f"{filename} must be list of users", file=sys.stderr)
+                sys.exit(1)
+
+            return {user["token"]: User(**user) for user in data}
+    except FileNotFoundError as e:
+        print(
+            f"File {filename} was not found, please make sure that you provide users file.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    except ValueError as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+
+users = load_users("/etc/api-server/users.yaml")
+
+
+def get_user(bearer_auth: HTTPAuthorizationCredentials = Security(bearer_auth)) -> User:
+    if bearer_auth and bearer_auth.credentials in users:
+        return users[bearer_auth.credentials]
+
+    raise HTTPException(
+        status_code=HTTP_403_FORBIDDEN,
+        detail="Could not validate credentials",
+    )
+
+
+class Link(BaseModel):
+    href: str
+
+
+class DeployArgs(BaseModel):
+    service: str
+    inventory: str | None = None
+    vars: Dict[str, Any] = {}
+
+
+class DeploymentLinks(BaseModel):
+    logs: Link
+    self: Link
+
+
+@dataclass
+class Deployment:
+    id: str
+    args: DeployArgs
+    runner: Runner | None = None
+
+
+class DeploymentDTO(BaseModel):
+    id: str
+    status: str
+    links: DeploymentLinks = Field(..., alias="_links")
+
+    def from_deployment(deployment: Deployment, request: Request):
+        return DeploymentDTO(
+            id=deployment.id,
+            status=deployment.runner.status,
+            _links={
+                "self": {"href": request.url_for("deployment", id=deployment.id)},
+                "logs": {"href": request.url_for("deployment_logs", id=deployment.id)},
+            },
+        )
+
+
+deployments: Dict[str, Deployment] = {}
+
+
+def get_deployment(id: str) -> Deployment:
+    if id not in deployments:
+        raise HTTPException(status_code=404, detail="Deployment was not found")
+
+    return deployments[id]
+
+
+app = FastAPI(
+    docs_url="/",
+    openapi_url="/swagger.json",
+    description="API for services management",
+    title="Server Management API",
+)
+
+
+@app.get("/deployment")
+async def deployment_list(request: Request, user: User = Depends(get_user)):
+    return [
+        DeploymentDTO.from_deployment(deployment, request)
+        for deployment in deployments.values()
+        if deployment.args.service in user.allowed_services
+    ]
+
+
+@app.get("/deployment/{id}/logs")
+async def deployment_logs(
+    deployment: Deployment = Depends(get_deployment), user: User = Depends(get_user)
+):
+    if deployment.args.service not in user.allowed_services:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail=f"This token does not allow to access {deployment.args.service} deployments.",
+        )
+
+    runner = deployment.runner
+
+    async def stream():
+        stdout = runner.stdout
+
+        while True:
+            while line := stdout.readline():
+                if line == "":
+                    break
+                yield line
+
+            if runner.status != "running":
+                break
+
+            await asyncio.sleep(0.1)
+
+    return StreamingResponse(stream(), media_type="text/plain")
+
+
+@app.get("/deployment/{id}")
+async def deployment(
+    request: Request,
+    deployment: Deployment = Depends(get_deployment),
+    user: User = Depends(get_user),
+):
+    if deployment.args.service not in user.allowed_services:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail=f"This token does not allow to access {deployment.args.service} deployments.",
+        )
+
+    return DeploymentDTO.from_deployment(deployments[id], request)
+
+
+@app.post("/deployment")
+async def deploy(args: DeployArgs, request: Request, user: User = Depends(get_user)):
+    if args.service not in user.allowed_services:
+        raise HTTPException(
+            status_code=HTTP_403_FORBIDDEN,
+            detail=f"This token does not allow to deploy {args.service} service.",
+        )
+
+    ident = str(uuid.uuid4())
+
+    _, runner = ansible_runner.run_async(
+        playbook="deploy.yaml",
+        ident=ident,
+        extravars={"services": [args.service], **args.vars},
+        private_data_dir="/var/run/ansible",
+        project_dir=os.environ.get("API_PROJECT_DIR", "/var/project"),
+        inventory=args.inventory or os.environ.get("API_INVENTORY"),
+        settings={"suppress_ansible_output": True},
+    )
+
+    deployment = Deployment(id=ident, runner=runner, args=args)
+    deployments[ident] = deployment
+
+    return DeploymentDTO.from_deployment(deployment, request)

api/bin/docker-entrypoint.d/00-check-project-mounted.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ -z "$(ls -A ${API_PROJECT_DIR})" ]; then
+    echo "No files found in project dir, maybe you forgot to mount it?" >&2
+    exit 1
+fi

api/bin/docker-entrypoint.d/01-proxy-ssh-agent.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+PROXIED_AUTH_SOCK="${PROXIED_AUTH_SOCK:-/var/run/proxied-ssh-auth.sock}"
+
+if [ -S "${PROXIED_AUTH_SOCK}" ]; then
+    echo "Found previously not closed ssh auth socket, closing."
+    rm ${PROXIED_AUTH_SOCK}
+fi
+
+# This should be run only when run as is specified and SSH Agent socket is present
+if [ -n "$API_RUNAS" ] && [ -S "$SSH_AUTH_SOCK" ]; then
+    echo "Proxying ${SSH_AUTH_SOCK} -> ${PROXIED_AUTH_SOCK} for ${API_RUNAS%%:*}"
+    socat UNIX-LISTEN:${PROXIED_AUTH_SOCK},fork,user=${API_RUNAS%%:*},mode=600 \
+          UNIX-CONNECT:${SSH_AUTH_SOCK} &
+    export SSH_AUTH_SOCK=/var/run/proxied-ssh-auth.sock
+fi 

api/bin/docker-entrypoint.d/02-make-private-directory-writable.sh

@@ -0,0 +1 @@
+chown $API_RUNAS /var/run/ansible

api/bin/docker-entrypoint.d/10-install-galaxy-requirements.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ -f "${API_GALAXY_REQUIREMENTS}" ]; then
+    echo "Installing galaxy stuff from ${API_GALAXY_REQUIREMENTS}"
+    run-user ansible-galaxy install -r ${API_GALAXY_REQUIREMENTS}
+fi

api/bin/docker-entrypoint.d/20-create-ansible-password-file.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ -n "$ANSIBLE_VAULT_PASSWORD" ]; then
+    export ANSIBLE_VAULT_PASSWORD_FILE=/var/run/secrets/vault-password
+    echo "$ANSIBLE_VAULT_PASSWORD" > $ANSIBLE_VAULT_PASSWORD_FILE
+    unset ANSIBLE_VAULT_PASSWORD
+fi 

api/bin/docker-entrypoint.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# fail on first error
+set -e
+
+function runuser {
+    EXEC=0
+
+    while true; do
+        case "$1" in
+        -E|--exec)
+            EXEC=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            break
+            ;;
+        esac
+    done
+
+    if [ -n "$API_RUNAS" ]; then
+        set -- su-exec "$API_RUNAS" "$@"
+    fi
+    
+    if [ $EXEC -eq 1 ]; then 
+        exec "$@"
+    else
+        "$@"
+    fi
+}
+
+alias run-user=runuser
+
+for part in $(dirname $0)/docker-entrypoint.d/*.sh; do 
+    [ -x $part ] && source $part
+done 
+
+if [ "${1#-}" != "$1" ] || [ $# -eq 0 ]; then
+    set -- uvicorn --app-dir /opt/api-server api:app --host ${API_HOST:-0.0.0.0} --port ${API_PORT:-8080} "$@"
+fi
+
+run-user --exec -- "$@"

api/fixtures/users.yaml

@@ -0,0 +1,2 @@
+- token: test
+  allowed_services: ["wipe-stg"]

api/requirements.txt

@@ -0,0 +1,4 @@
+fastapi>=0.68.0,<0.69.0
+pydantic>=1.8.0,<2.0.0
+uvicorn>=0.15.0,<0.16.0
+ansible-runner>=2.3.0,<2.4.0

docker-compose.yaml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  api:
+    build: api
+    image: registry.kadet.net/management/api-server:${API_VERSION:-latest}
+    environment:
+      - SSH_AUTH_SOCK
+    ports: 
+      - "8080:8080"
+    command: ['--reload', '--reload-dir', '/opt/api-server']
+    volumes:
+      - .:/var/project
+      - ./api:/opt/api-server
+      - ./api/fixtures/users.yaml:/etc/api-server/users.yaml:ro
+      - ${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}

galaxy-requirements.yml

@@ -0,0 +1,13 @@
+---
+roles:
+  - name: geerlingguy.docker
+    version: 3.0.0
+  - name: geerlingguy.pip
+    version: 2.0.0
+  - name: geerlingguy.mysql
+    version: 3.3.0
+collections:
+  - name: ansible.posix
+    version: 1.4.0
+  - name: community.docker
+    version: 2.7.1

main.yaml

@@ -2,27 +2,8 @@
 - hosts: all
   become: yes
 
-  vars_files:
-    - vars/services.yml
-    - vars/environment.yml
-    - vars/databases.yml
-
-  roles:
-    - geerlingguy.docker
-    - geerlingguy.pip
-    - geerlingguy.mysql
-    - kadet.docker-swarm
-
-  tasks:
-    - name: "Add '{{ ansible_user }}' to docker group"
-      user:
-        user: "{{ ansible_user }}"
-        groups: docker
-        append: yes
-    - name: "Add acl package"
-      apt: 
-        name: acl
-        state: present
+- name: Setup server
+  import_playbook: setup.yaml
 
 - name: Deploy services
   import_playbook: deploy.yaml

ping.yaml

@@ -0,0 +1,4 @@
+- hosts:
+    - all
+  tasks:
+    - action: ping

services/management/stack.yml

@@ -0,0 +1,56 @@
+version: "{{ compose_version }}"
+
+services:
+  api:
+    image: registry.kadet.net/management/api-server:{{ api_server_version }}
+    command: ['--proxy-headers']
+    extra_hosts:
+      - manager.swarm.local:{{ swarm_host_address }}
+    networks:
+      - default
+      - "{{ ingress_network }}"
+    environment:
+      - API_INVENTORY=inventory/swarm.ini
+      - ANSIBLE_VAULT_PASSWORD_FILE=/var/run/secrets/vault-password
+    volumes:
+      - ./project:/var/project
+      - private-dir:/var/run/ansible
+    secrets:
+      - source: id-rsa
+        target: /home/api-server/.ssh/id_rsa
+      - source: users_{{ users_config.checksum }}
+        target: /etc/api-server/users.yaml
+      - source: vault-password_{{ vault_password.checksum[:12] }}
+        target: /var/run/secrets/vault-password
+    configs:
+      - source: id-rsa-pub
+        target: /home/api-server/.ssh/id_rsa.pub
+      - source: inventory_{{ inventory_config.checksum }}
+        target: /var/project/inventory/swarm.ini
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.{{ service }}.rule=Host(`mgmt.{{ main_domain }}`)
+        - traefik.http.routers.{{ service }}.tls=true
+        - traefik.http.routers.{{ service }}.tls.certresolver=lets-encrypt
+        - traefik.http.services.{{ service }}.loadbalancer.server.port=8080
+      placement:
+        constraints:
+          - node.role == manager
+
+volumes:
+  private-dir: ~
+
+configs:
+  id-rsa-pub:
+    file: ./ssh/id_rsa.pub
+  inventory_{{ inventory_config.checksum }}:
+    file: ./config/inventory.ini
+
+secrets:
+  id-rsa:
+    file: ./ssh/id_rsa
+  users_{{ users_config.checksum }}:
+    file: ./config/users.yaml
+  vault-password_{{ vault_password.checksum[:12] }}:
+    file: ./config/vault-password

services/management/tasks/config.yml

@@ -0,0 +1,45 @@
+---
+- name: 'Ensure config directory exists for "{{ service }}"'
+  file:
+    path: "{{ remote_service_path }}/config"
+    state: directory
+    owner: "{{ ansible_user }}"
+  tags:
+    - config
+
+- name: 'Generate inventory file for "{{ service }}"'
+  template:
+    src: "{{ service_path }}/templates/inventory.ini.j2"
+    dest: "{{ remote_service_path }}/config/inventory.ini"
+  register: inventory_config
+  tags:
+    - config
+
+- name: 'Generate users file for "{{ service }}"'
+  template:
+    src: "{{ service_path }}/templates/users.yml.j2"
+    dest: "{{ remote_service_path }}/config/users.yaml"
+  register: users_config
+  tags:
+    - config
+
+- name: 'Generate vault password file for "{{ service }}"'
+  when: api_server_vault_password is defined
+  copy:
+    dest: "{{ remote_service_path }}/config/vault-password"
+    content: "{{ api_server_vault_password }}"
+  register: vault_password
+  tags:
+    - config
+
+- name: 'Ensure SSH config directory exists for "{{ service }}'
+  file:
+    path: "{{ remote_service_path }}/ssh"
+    state: directory
+    owner: "{{ ansible_user }}"
+  tags:
+    - config
+
+- name: 'Generate SSH key pair for "{{ service }}"'
+  community.crypto.openssh_keypair:
+    path: "{{ remote_service_path }}/ssh/id_rsa"

services/management/templates/inventory.ini.j2

@@ -0,0 +1,7 @@
+[main]
+manager.swarm.local
+
+[main:vars]
+ansible_user={{ ansible_user }}
+main_domain={{ main_domain }}
+swarm_addr={{ swarm_addr }}

services/management/templates/users.yml.j2

@@ -0,0 +1 @@
+{{ api_server_users|to_nice_yaml(indent=2) }}

services/management/vars/main.yml

@@ -0,0 +1,3 @@
+---
+api_server_version: latest
+api_server_users: []

services/system/stack.yml

@@ -26,3 +26,16 @@ services:
         - "swarm.cronjob.schedule=0 0 0 * * sun"
       restart_policy:
         condition: none
+
+  volume-cleanup-job:
+    image: docker
+    command: ["docker", "volume", "prune", "-f"]
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    deploy:
+      mode: global
+      labels:
+        - "swarm.cronjob.enable=true"
+        - "swarm.cronjob.schedule=0 0 0 * * *"
+      restart_policy:
+        condition: none

setup.yaml

@@ -0,0 +1,26 @@
+---
+- hosts: all
+  become: yes
+
+  vars_files:
+    - vars/services.yml
+    - vars/environment.yml
+    - vars/databases.yml
+
+  roles:
+    - geerlingguy.docker
+    - geerlingguy.pip
+    - geerlingguy.mysql
+    - kadet.docker-swarm
+
+  tasks:
+    - name: "Add '{{ ansible_user }}' to docker group"
+      user:
+        user: "{{ ansible_user }}"
+        groups: docker
+        append: yes
+
+    - name: "Add acl package"
+      apt: 
+        name: acl
+        state: present

vars/services.yml

@@ -142,6 +142,15 @@ service_config:
           6361376166366438640a323166363063373033356466633839316433613566643734633930363766
           30633736373161383238303262393635393436393637323639366135323530316666623030343633
           6131663563663936316632373565363566343364613666363366
+      - name: github-actions
+        password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          30643365613763383464393263636165373331636139626137376231646536336339613861376631
+          3538383539346566626330326365666164313531336132300a616434623133396665373565353130
+          33366163633136653666343363653464333136626262396337376563623839316536666161373230
+          6562323935356463620a626664313863383730656137383833313766656461386337646531643864
+          61353763393838326561366330653562343133363534656335326332643632643065663437316139
+          6435376534383463346639656261383632323639373930333961
     registry_storage:
       s3:
         accesskey: !vault |
@@ -161,7 +170,28 @@ service_config:
         region: eu-central-003
         regionendpoint: https://s3.eu-central-003.backblazeb2.com
         bucket: kadet-docker
-  portainer: ~
-  system: ~
+  portainer: {}
+  system: {}
+  management:
+    api_server_vault_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          38376138373336636630386339653761646333313238386365653365623335383035643231643162
+          3936353164363934633939306462346262666438346262310a313366353766303833373734383032
+          38643039376636633762653838623565376236653061633734626262616530313734366231326632
+          6530373935323033360a613236366135656436306330306162636661313963613466656163316139
+          32333032613461633834396261623166656365666364393761326438366536306237373661303834
+          3339396235393636346666346365306463643430373639613762
+    api_server_users:
+      - token: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          66663262646661623837363865366463323034376436303738353165306464623933393464666433
+          6632356435323463303436333561333030383163646630350a613831303734336234623366313931
+          39623963653466326239376163313033313139653332343738306634623036666464326131633532
+          3730643635373436380a396662643061653664653366356137366538653431373361383236336362
+          34366665636631373165346338343962613266643036386532326461633766633632373932383663
+          32363837663436663131306238616536623637376137393134623565353436663864333666346231
+          38333435346561306163666533373166363739636533383335663435363431643566383930366163
+          31363565343065313631
+        allowed_services: ["wipe-stg"]
 www_data_users:
   - vagrant