kadet.net/servers
1
0
Fork 0
Code Releases Activity

cojedzie: Add ClickHouse for the departure history

Browse Source 
Each stack gets its own ClickHouse (2G limit, system logs off, low-memory config) and the schedule worker consumes scheduler_recorder. Next keeps 30 days of history, production 365. Goes live with the move to the bigger instance.
This commit is contained in:
Kacper Donat 2026-11-01 13:51:26 +01:00
parent fe5b55554d
commit e11a99cb2a
11 changed files with 169 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
services/cojedzie/config/clickhouse.xml Normal file
View File

@ -0,0 +1,30 @@
<clickhouse>
<!-- a guest on a shared box, not a dedicated server -->
<max_server_memory_usage>1610612736</max_server_memory_usage>
<mark_cache_size>268435456</mark_cache_size>
<uncompressed_cache_size>0</uncompressed_cache_size>
<!-- containers often have no IPv6, listening on :: fails noisily -->
<listen_host>0.0.0.0</listen_host>
<logger>
<level>warning</level>
<console>1</console>
</logger>
<!-- internal system logs grow without limit; we don't use them -->
<query_log remove="1"/>
<query_thread_log remove="1"/>
<query_views_log remove="1"/>
<query_metric_log remove="1"/>
<trace_log remove="1"/>
<metric_log remove="1"/>
<asynchronous_metric_log remove="1"/>
<part_log remove="1"/>
<text_log remove="1"/>
<processors_profile_log remove="1"/>
<opentelemetry_span_log remove="1"/>
<session_log remove="1"/>
<error_log remove="1"/>
<latency_log remove="1"/>
<backup_log remove="1"/>
<s3queue_log remove="1"/>
<blob_storage_log remove="1"/>
</clickhouse>

32
services/cojedzie/stack.yml
View File

@ -42,11 +42,36 @@ services:
<<: *worker <<: *worker
environment: environment:
COJEDZIE_WORKER_OPTS: '-vv' COJEDZIE_WORKER_OPTS: '-vv'
COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub' COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub scheduler_recorder'
worker: worker:
<<: *worker <<: *worker
# departure history (cojedzie ADR 0020); reached only inside the stack
clickhouse:
image: clickhouse/clickhouse-server:{{ cojedzie_clickhouse_version|default('25.8') }}
environment:
CLICKHOUSE_DB: cojedzie
CLICKHOUSE_USER: cojedzie
CLICKHOUSE_PASSWORD: "{{ cojedzie_clickhouse_password }}"
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- clickhouse_data:/var/lib/clickhouse
configs:
- source: clickhouse_config_{{ clickhouse_config.checksum }}
target: /etc/clickhouse-server/config.d/low-memory.xml
mode: 0444
deploy:
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
mercure: mercure:
image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }} image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }}
networks: networks:
@ -94,7 +119,12 @@ services:
- traefik.http.routers.{{ service }}-front.middlewares=gzip@file - traefik.http.routers.{{ service }}-front.middlewares=gzip@file
- traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000 - traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000
configs:
clickhouse_config_{{ clickhouse_config.checksum }}:
file: ./config/clickhouse.xml
volumes: volumes:
db_data: ~ db_data: ~
clickhouse_data: ~
mercure_data: ~ mercure_data: ~
mercure_config: ~ mercure_config: ~

8
services/cojedzie/tasks/config.yaml
View File

@ -11,3 +11,11 @@
- api.env - api.env
- front.env - front.env
- sentry.env - sentry.env
- name: 'Copy ClickHouse config for "{{ service }}"'
template:
src: "{{ service_path }}/config/clickhouse.xml"
dest: "{{ remote_service_path }}/config/clickhouse.xml"
register: clickhouse_config
tags:
- config

4
services/cojedzie/templates/api.env
View File

@ -8,3 +8,7 @@ COJEDZIE_DATABASE_URL=mysql://{{ cojedzie_database.user }}:{{ cojedzie_database.
SENTRY_DSN={{ sentry_dsn_api }} SENTRY_DSN={{ sentry_dsn_api }}
SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }} SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }}
# departure history; see cojedzie's contrib/agents/wiki/api/departure-history.md
COJEDZIE_CLICKHOUSE_URL=http://cojedzie:{{ cojedzie_clickhouse_password | urlencode }}@clickhouse:8123/cojedzie
COJEDZIE_DEPARTURE_HISTORY_RETENTION_DAYS={{ cojedzie_departure_history_retention_days }}

1
services/cojedzie/vars/main.yml
View File

@ -24,3 +24,4 @@ sentry_session_replay_rate: 0.01
sentry_error_replay_rate: 0.25 sentry_error_replay_rate: 0.25
sentry_environment: production sentry_environment: production
traefik_routing_rule: "Host(`{{ cojedzie_domain }}`) || Host(`{{ service }}.{{ main_domain }}`)" traefik_routing_rule: "Host(`{{ cojedzie_domain }}`) || Host(`{{ service }}.{{ main_domain }}`)"
cojedzie_departure_history_retention_days: 365

30
services/cojedzie@next/config/clickhouse.xml Normal file
View File

@ -0,0 +1,30 @@
<clickhouse>
<!-- a guest on a shared box, not a dedicated server -->
<max_server_memory_usage>1610612736</max_server_memory_usage>
<mark_cache_size>268435456</mark_cache_size>
<uncompressed_cache_size>0</uncompressed_cache_size>
<!-- containers often have no IPv6, listening on :: fails noisily -->
<listen_host>0.0.0.0</listen_host>
<logger>
<level>warning</level>
<console>1</console>
</logger>
<!-- internal system logs grow without limit; we don't use them -->
<query_log remove="1"/>
<query_thread_log remove="1"/>
<query_views_log remove="1"/>
<query_metric_log remove="1"/>
<trace_log remove="1"/>
<metric_log remove="1"/>
<asynchronous_metric_log remove="1"/>
<part_log remove="1"/>
<text_log remove="1"/>
<processors_profile_log remove="1"/>
<opentelemetry_span_log remove="1"/>
<session_log remove="1"/>
<error_log remove="1"/>
<latency_log remove="1"/>
<backup_log remove="1"/>
<s3queue_log remove="1"/>
<blob_storage_log remove="1"/>
</clickhouse>

32
services/cojedzie@next/stack.yml
View File

@ -42,11 +42,36 @@ services:
<<: *worker <<: *worker
environment: environment:
COJEDZIE_WORKER_OPTS: '-vv' COJEDZIE_WORKER_OPTS: '-vv'
COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub' COJEDZIE_WORKER_QUEUES: 'scheduler_default scheduler_hub scheduler_recorder'
worker: worker:
<<: *worker <<: *worker
# departure history (cojedzie ADR 0020); reached only inside the stack
clickhouse:
image: clickhouse/clickhouse-server:{{ cojedzie_clickhouse_version|default('25.8') }}
environment:
CLICKHOUSE_DB: cojedzie
CLICKHOUSE_USER: cojedzie
CLICKHOUSE_PASSWORD: "{{ cojedzie_clickhouse_password }}"
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- clickhouse_data:/var/lib/clickhouse
configs:
- source: clickhouse_config_{{ clickhouse_config.checksum }}
target: /etc/clickhouse-server/config.d/low-memory.xml
mode: 0444
deploy:
mode: replicated
replicas: 1
resources:
limits:
memory: 2G
mercure: mercure:
image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }} image: dunglas/mercure:{{ cojedzie_mercure_version|default('latest') }}
networks: networks:
@ -94,7 +119,12 @@ services:
- traefik.http.routers.{{ service }}-front.middlewares=gzip@file - traefik.http.routers.{{ service }}-front.middlewares=gzip@file
- traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000 - traefik.http.services.{{ service }}-front.loadbalancer.server.port=3000
configs:
clickhouse_config_{{ clickhouse_config.checksum }}:
file: ./config/clickhouse.xml
volumes: volumes:
db_data: ~ db_data: ~
clickhouse_data: ~
mercure_data: ~ mercure_data: ~
mercure_config: ~ mercure_config: ~

8
services/cojedzie@next/tasks/config.yaml
View File

@ -11,3 +11,11 @@
- api.env - api.env
- front.env - front.env
- sentry.env - sentry.env
- name: 'Copy ClickHouse config for "{{ service }}"'
template:
src: "{{ service_path }}/config/clickhouse.xml"
dest: "{{ remote_service_path }}/config/clickhouse.xml"
register: clickhouse_config
tags:
- config

4
services/cojedzie@next/templates/api.env
View File

@ -8,3 +8,7 @@ COJEDZIE_DATABASE_URL=mysql://{{ cojedzie_database.user }}:{{ cojedzie_database.
SENTRY_DSN={{ sentry_dsn_api }} SENTRY_DSN={{ sentry_dsn_api }}
SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }} SENTRY_SAMPLE_RATE={{ sentry_sample_rate_api }}
# departure history; see cojedzie's contrib/agents/wiki/api/departure-history.md
COJEDZIE_CLICKHOUSE_URL=http://cojedzie:{{ cojedzie_clickhouse_password | urlencode }}@clickhouse:8123/cojedzie
COJEDZIE_DEPARTURE_HISTORY_RETENTION_DAYS={{ cojedzie_departure_history_retention_days }}

1
services/cojedzie@next/vars/main.yml
View File

@ -28,3 +28,4 @@ sentry_error_replay_rate: 0.25
sentry_environment: production sentry_environment: production
traefik_routing_rule: "Host(`{{ cojedzie_domain }}`)" traefik_routing_rule: "Host(`{{ cojedzie_domain }}`)"
cojedzie_departure_history_retention_days: 365

21
vars/services.yml
View File

@ -191,6 +191,16 @@ service_config:
user: cojedzie user: cojedzie
password: "{{ mysql_user_passwords['cojedzie'] }}" password: "{{ mysql_user_passwords['cojedzie'] }}"
cojedzie_clickhouse_password: !vault |
$ANSIBLE_VAULT;1.1;AES256
63643266343539623336316133643238383261323738313364376464653430663737653864633461
6333303062616532393632313662656264653037326463370a363163633736313366663762636364
31643737383036346466303037326438643365616636386438316632373065643635333566336236
6537366634303937300a646163666564316261636238363864306337643039623638313136666434
63373065363634313539353638613263313131666631313365613439396666613436346566633239
62376431396233346538623733643631343664323663333765373364663636646236393234313132
383262633465306566663130346237333737
cojedzie-next: cojedzie-next:
template: cojedzie@next template: cojedzie@next
@ -213,6 +223,17 @@ service_config:
user: cojedzie-next user: cojedzie-next
password: "{{ mysql_user_passwords['cojedzie-next'] }}" password: "{{ mysql_user_passwords['cojedzie-next'] }}"
cojedzie_clickhouse_password: !vault |
$ANSIBLE_VAULT;1.1;AES256
33396439313166623461353264393734356533626564623730663366366636306536353031623934
6132393935343136353337326237626563373162353433310a383663336462613563666334626234
64343161353239346261633566393230376562373237616638366233363230393734376338383534
3434633537356463330a323132633030336263343238323934376561336362643464323766363830
66326366663764393464316166653363336661346334623865306564353738356264633564626633
36313531636437336236303832333736383434323632633966663632333166663732633162613538
643335646537333031356134336434613430
cojedzie_departure_history_retention_days: 30
sentry_environment: next sentry_environment: next
sentry_sample_rate_front: 0.01 sentry_sample_rate_front: 0.01
sentry_sample_rate_api: 0.01 sentry_sample_rate_api: 0.01