initial commit

Makefile

@@ -0,0 +1,12 @@
+SHELL = /bin/bash -o pipefail
+
+ROOT_DIR=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+MIGRATIONS_DIR=$(SRC_DIR)/files/migrations
+
+migrate:
+	@docker-compose up -d
+	@docker run -v $(MIGRATIONS_DIR):/migrations --network host migrate/migrate -path=/migrations/ -database "clickhouse://localhost:8123?username=default&password=default&database=default" up all
+	@docker-compose stop
+
+up:
+	@docker-compose up

docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3.8'
+
+services:
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - grafana_data:/var/lib/grafana
+    ports:
+      - '3000:3000'
+  clickhouse:
+    image: docker.io/bitnami/clickhouse:23
+    environment:
+      - CLICKHOUSE_ADMIN_USER=default
+      - CLICKHOUSE_ADMIN_PASSWORD=default
+    ports:
+      - '8123:8123'
+    volumes:
+      - clickhouse_data:/bitnami/clickhouse
+  vector:
+    image: timberio/vector:latest-alpine
+    ports:
+      - '8686:8686'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./files/vector.toml:/etc/vector/vector.toml:ro
+
+volumes:
+  clickhouse_data:
+    driver: local
+  grafana_data:
+    driver: local

files/migrations/1698135711_create_logs_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE log_docker_raw;

files/migrations/1698135711_create_logs_table.up.sql

@@ -0,0 +1,7 @@
+CREATE TABLE log_docker_raw (
+  host String,
+  logdatetime DateTime PRIMARY KEY,
+  message String,
+  priority String,
+  program String
+) Engine = MergeTree();

files/vector.toml

@@ -0,0 +1,38 @@
+[api]
+enabled = true
+address = "0.0.0.0:8686"
+
+[sources.docker]
+type = "docker_logs"
+exclude_containers = [ "clickhouse", "grafana", "vector" ]
+
+[transforms.ts_and_program]
+type = "remap"
+inputs = [ "docker" ]
+source = '''
+  .logdatetime = to_unix_timestamp(now())
+  .program = .container_name
+'''
+
+[transforms.emptyprogram]
+type = "filter"
+inputs = [ "ts_and_program" ]
+condition.type = "vrl"
+condition.source = '''
+  .program != ""
+'''
+
+[transforms.dedupe]
+type = "dedupe"
+inputs = [ "emptyprogram" ]
+
+[sinks.clickhouse]
+type = "clickhouse"
+inputs = [ "dedupe"] 
+compression = "gzip"
+endpoint = "http://clickhouse:8123"
+auth.strategy = "basic"
+auth.user = "default"
+auth.password = "default"
+database = "default"
+table = "log_docker_raw"