cojedzie: Add ClickHouse for the departure history

Each stack gets its own ClickHouse (2G limit, system logs off, low-memory config) and the schedule worker consumes scheduler_recorder. Next keeps 30 days of history, production 365. Goes live with the move to the bigger instance.

services/cojedzie/config/clickhouse.xml

@@ -0,0 +1,30 @@
+<clickhouse>
+    <!-- a guest on a shared box, not a dedicated server -->
+    <max_server_memory_usage>1610612736</max_server_memory_usage>
+    <mark_cache_size>268435456</mark_cache_size>
+    <uncompressed_cache_size>0</uncompressed_cache_size>
+    <!-- containers often have no IPv6, listening on :: fails noisily -->
+    <listen_host>0.0.0.0</listen_host>
+    <logger>
+        <level>warning</level>
+        <console>1</console>
+    </logger>
+    <!-- internal system logs grow without limit; we don't use them -->
+    <query_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <query_views_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <trace_log remove="1"/>
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <text_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <session_log remove="1"/>
+    <error_log remove="1"/>
+    <latency_log remove="1"/>
+    <backup_log remove="1"/>
+    <s3queue_log remove="1"/>
+    <blob_storage_log remove="1"/>
+</clickhouse>

services/cojedzie/stack.yml

@@ -42,11 +42,36 @@ services:
     <<: *worker
     environment:
       COJEDZIE_WORKER_OPTS: '-vv'
-      COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub'
+      COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub scheduler_recorder'
 
   worker:
     <<: *worker
 
+  # departure history (cojedzie ADR 0020); reached only inside the stack
+  clickhouse:
+    image: clickhouse/clickhouse-server:{{ cojedzie_clickhouse_version|default('25.8') }}
+    environment:
+      CLICKHOUSE_DB: cojedzie
+      CLICKHOUSE_USER: cojedzie
+      CLICKHOUSE_PASSWORD: "{{ cojedzie_clickhouse_password }}"
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    configs:
+      - source: clickhouse_config_{{ clickhouse_config.checksum }}
+        target: /etc/clickhouse-server/config.d/low-memory.xml
+        mode: 0444
+    deploy:
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          memory: 2G
+
   mercure:
     image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }}
     networks:
@@ -94,7 +119,12 @@ services:
         - traefik.http.routers.{{ service }}-front.middlewares=gzip@file
         - traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000
 
+configs:
+  clickhouse_config_{{ clickhouse_config.checksum }}:
+    file: ./config/clickhouse.xml
+
 volumes:
   db_data: ~
+  clickhouse_data: ~
   mercure_data: ~
   mercure_config: ~

services/cojedzie/tasks/config.yaml

@@ -11,3 +11,11 @@
     - api.env
     - front.env
     - sentry.env
+
+- name: 'Copy ClickHouse config for "{{ service }}"'
+  template:
+    src: "{{ service_path }}/config/clickhouse.xml"
+    dest: "{{ remote_service_path }}/config/clickhouse.xml"
+  register: clickhouse_config
+  tags:
+    - config

services/cojedzie/templates/api.env

@@ -8,3 +8,7 @@ COJEDZIE_DATABASE_URL=mysql://{{ cojedzie_database.user }}:{{ cojedzie_database.
 
 SENTRY_DSN={{ sentry_dsn_api }}
 SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }}
+
+# departure history; see cojedzie's contrib/agents/wiki/api/departure-history.md
+COJEDZIE_CLICKHOUSE_URL=http://cojedzie:{{ cojedzie_clickhouse_password | urlencode }}@clickhouse:8123/cojedzie
+COJEDZIE_DEPARTURE_HISTORY_RETENTION_DAYS={{ cojedzie_departure_history_retention_days }}

services/cojedzie/vars/main.yml

@@ -24,3 +24,4 @@ sentry_session_replay_rate: 0.01
 sentry_error_replay_rate: 0.25
 sentry_environment: production
 traefik_routing_rule: "Host(`{{ cojedzie_domain }}`) || Host(`{{ service }}.{{ main_domain }}`)"
+cojedzie_departure_history_retention_days: 365

services/cojedzie@next/config/clickhouse.xml

@@ -0,0 +1,30 @@
+<clickhouse>
+    <!-- a guest on a shared box, not a dedicated server -->
+    <max_server_memory_usage>1610612736</max_server_memory_usage>
+    <mark_cache_size>268435456</mark_cache_size>
+    <uncompressed_cache_size>0</uncompressed_cache_size>
+    <!-- containers often have no IPv6, listening on :: fails noisily -->
+    <listen_host>0.0.0.0</listen_host>
+    <logger>
+        <level>warning</level>
+        <console>1</console>
+    </logger>
+    <!-- internal system logs grow without limit; we don't use them -->
+    <query_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <query_views_log remove="1"/>
+    <query_metric_log remove="1"/>
+    <trace_log remove="1"/>
+    <metric_log remove="1"/>
+    <asynchronous_metric_log remove="1"/>
+    <part_log remove="1"/>
+    <text_log remove="1"/>
+    <processors_profile_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <session_log remove="1"/>
+    <error_log remove="1"/>
+    <latency_log remove="1"/>
+    <backup_log remove="1"/>
+    <s3queue_log remove="1"/>
+    <blob_storage_log remove="1"/>
+</clickhouse>

services/cojedzie@next/stack.yml

@@ -42,11 +42,36 @@ services:
     <<: *worker
     environment:
       COJEDZIE_WORKER_OPTS: '-vv'
-      COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub'
+      COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub scheduler_recorder'
 
   worker:
     <<: *worker
 
+  # departure history (cojedzie ADR 0020); reached only inside the stack
+  clickhouse:
+    image: clickhouse/clickhouse-server:{{ cojedzie_clickhouse_version|default('25.8') }}
+    environment:
+      CLICKHOUSE_DB: cojedzie
+      CLICKHOUSE_USER: cojedzie
+      CLICKHOUSE_PASSWORD: "{{ cojedzie_clickhouse_password }}"
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    configs:
+      - source: clickhouse_config_{{ clickhouse_config.checksum }}
+        target: /etc/clickhouse-server/config.d/low-memory.xml
+        mode: 0444
+    deploy:
+      mode: replicated
+      replicas: 1
+      resources:
+        limits:
+          memory: 2G
+
   mercure:
     image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }}
     networks:
@@ -94,7 +119,12 @@ services:
         - traefik.http.routers.{{ service }}-front.middlewares=gzip@file
         - traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000
 
+configs:
+  clickhouse_config_{{ clickhouse_config.checksum }}:
+    file: ./config/clickhouse.xml
+
 volumes:
   db_data: ~
+  clickhouse_data: ~
   mercure_data: ~
   mercure_config: ~

services/cojedzie@next/tasks/config.yaml

@@ -11,3 +11,11 @@
     - api.env
     - front.env
     - sentry.env
+
+- name: 'Copy ClickHouse config for "{{ service }}"'
+  template:
+    src: "{{ service_path }}/config/clickhouse.xml"
+    dest: "{{ remote_service_path }}/config/clickhouse.xml"
+  register: clickhouse_config
+  tags:
+    - config

services/cojedzie@next/templates/api.env

@@ -8,3 +8,7 @@ COJEDZIE_DATABASE_URL=mysql://{{ cojedzie_database.user }}:{{ cojedzie_database.
 
 SENTRY_DSN={{ sentry_dsn_api }}
 SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }}
+
+# departure history; see cojedzie's contrib/agents/wiki/api/departure-history.md
+COJEDZIE_CLICKHOUSE_URL=http://cojedzie:{{ cojedzie_clickhouse_password | urlencode }}@clickhouse:8123/cojedzie
+COJEDZIE_DEPARTURE_HISTORY_RETENTION_DAYS={{ cojedzie_departure_history_retention_days }}

services/cojedzie@next/vars/main.yml

@@ -28,3 +28,4 @@ sentry_error_replay_rate: 0.25
 sentry_environment: production
 
 traefik_routing_rule: "Host(`{{ cojedzie_domain }}`)"
+cojedzie_departure_history_retention_days: 365

vars/services.yml

@@ -191,6 +191,16 @@ service_config:
       user: cojedzie
       password: "{{ mysql_user_passwords['cojedzie'] }}"
 
+    cojedzie_clickhouse_password: !vault |
+              $ANSIBLE_VAULT;1.1;AES256
+              63643266343539623336316133643238383261323738313364376464653430663737653864633461
+              6333303062616532393632313662656264653037326463370a363163633736313366663762636364
+              31643737383036346466303037326438643365616636386438316632373065643635333566336236
+              6537366634303937300a646163666564316261636238363864306337643039623638313136666434
+              63373065363634313539353638613263313131666631313365613439396666613436346566633239
+              62376431396233346538623733643631343664323663333765373364663636646236393234313132
+              383262633465306566663130346237333737
+
   cojedzie-next:
     template: cojedzie@next
 
@@ -213,6 +223,17 @@ service_config:
       user: cojedzie-next
       password: "{{ mysql_user_passwords['cojedzie-next'] }}"
 
+    cojedzie_clickhouse_password: !vault |
+              $ANSIBLE_VAULT;1.1;AES256
+              33396439313166623461353264393734356533626564623730663366366636306536353031623934
+              6132393935343136353337326237626563373162353433310a383663336462613563666334626234
+              64343161353239346261633566393230376562373237616638366233363230393734376338383534
+              3434633537356463330a323132633030336263343238323934376561336362643464323766363830
+              66326366663764393464316166653363336661346334623865306564353738356264633564626633
+              36313531636437336236303832333736383434323632633966663632333166663732633162613538
+              643335646537333031356134336434613430
+    cojedzie_departure_history_retention_days: 30
+
     sentry_environment: next
     sentry_sample_rate_front: 0.01
     sentry_sample_rate_api: 0.01