Neur0toxine/docker-logs-grafana-clickhouse
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Neur0toxine:3a8a0fb79830bac9077656b29df3c50ef2e26298
Branches Tags
Neur0toxine:master
...
compare: Neur0toxine:f3382f71f312e6e6981d71357e3d16fbabdeaffe
Branches Tags
Neur0toxine:master

5 Commits
3a8a0fb798 ... f3382f71f3

Author SHA1 Message Date
Neur0toxine
f3382f71f3 Merge branch 'master' of git.neur0tx.site:Neur0toxine/docker-logs-grafana-clickhouse 2024-09-26 12:50:48 +03:00
Neur0toxine
61c6113189
Update README.md 2024-09-26 12:48:22 +03:00
Neur0toxine
e770437dec update dashboard, several fixes 2024-09-26 12:47:00 +03:00
Neur0toxine
c9f0ac9595 update format 2024-05-28 14:04:03 +03:00
Neur0toxine
a4ed616503 update 2024-05-21 14:56:35 +03:00
5 changed files with 486 additions and 550 deletions
Show Stats Expand all files Collapse all files

8
README.md
View File

@ -1,6 +1,6 @@
# Grafana Clickhouse Docker Log Aggregation # Grafana Clickhouse Docker Log Aggregation
This demo project can aggregate logs from Docker containers and store them in Clickhouse. In the `dashboards` directory you can find dashboard examples for specific log format. `Logs + Structured via Query` is recommended. This demo project can aggregate logs from Docker containers and store them in Clickhouse. In the `dashboards` directory you can find dashboard examples for specific log format. `Logs + Structured via Query` is recommended because the other one is deprecated now.
**Note**: dashboards are tuned for structured logging (JSON log records). **Note**: dashboards are tuned for structured logging (JSON log records).
@ -8,8 +8,8 @@ This demo project can aggregate logs from Docker containers and store them in Cl
Obviously, you'll need Docker: Obviously, you'll need Docker:
1. `make all` 1. `make migrate` (use `clickhouse-client` inside container and execute migration SQL directly if migrator doesn't work).
2. Navigate to `http://grafana.test` or `http://localhost:3000`, use `admin:admin` as credentials. 2. Navigate to `http://grafana.test` or `http://localhost:3000` (if you did not configure Traefik locally), use `admin:admin` as credentials.
3. Create your own dashboard or import those I provided (`dashboards/` directory). Replace data source UUID with correct one if needed. 3. Import `dashboards/Logs + Structured via Query.json` dashboard.
4. Run any Docker project with structured JSON logs. 4. Run any Docker project with structured JSON logs.
5. Done. 5. Done.

53
compose.yml Normal file
View File

@ -0,0 +1,53 @@
services:
grafana:
image: grafana/grafana:latest
volumes:
- grafana_data:/var/lib/grafana
depends_on:
- clickhouse
ports:
- 3000:3000
labels:
traefik.enable: "true"
traefik.http.routers.grafana.entrypoints: web
traefik.http.routers.grafana.rule: 'Host(`grafana.test`)'
traefik.http.services.grafana.loadbalancer.server.port: 3000
clickhouse:
image: clickhouse/clickhouse-server:24
environment:
- CLICKHOUSE_DB=default
- CLICKHOUSE_USER=default
- CLICKHOUSE_PASSWORD=default
- CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
ports:
- '8123:8123'
ulimits:
nofile:
soft: 262144
hard: 262144
cap_add:
- NET_ADMIN
- SYS_NICE
- IPC_LOCK
healthcheck:
test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
interval: 10s
timeout: 5s
retries: 5
volumes:
- clickhouse_data:/var/lib/clickhouse
vector:
image: timberio/vector:latest-alpine
ports:
- '8686:8686'
depends_on:
- clickhouse
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes:
clickhouse_data:
driver: local
grafana_data:
driver: local

451
dashboards/Logs + Structured via Query.json
View File

@ -38,7 +38,7 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"description": "", "description": "",
"fieldConfig": { "fieldConfig": {
@ -53,6 +53,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 0, "fillOpacity": 0,
"gradientMode": "hue", "gradientMode": "hue",
@ -157,22 +158,6 @@
"operator": "LIKE", "operator": "LIKE",
"type": "String", "type": "String",
"value": "$filter1" "value": "$filter1"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter2"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter3"
} }
], ],
"limit": 5000, "limit": 5000,
@ -191,10 +176,10 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"queryType": "builder", "queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) AND ( message LIKE '%$filter2%' ) AND ( message LIKE '%$filter3%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
"refId": "A" "refId": "A"
} }
], ],
@ -204,13 +189,12 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {
"fixedColor": "red", "mode": "palette-classic"
"mode": "fixed"
}, },
"custom": { "custom": {
"axisBorderShow": false, "axisBorderShow": false,
@ -219,6 +203,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 0, "fillOpacity": 0,
"gradientMode": "none", "gradientMode": "none",
@ -249,8 +234,12 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "red", "color": "green",
"value": null "value": null
},
{
"color": "red",
"value": 80
} }
] ]
} }
@ -326,61 +315,11 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"format": 1, "queryType": "builder",
"meta": { "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
"builderOptions": { "refId": "A"
"database": "default",
"fields": [
"priority"
],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "logdatetime",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "logdatetime",
"name": "logdatetime",
"picklistValues": [],
"type": "DateTime"
}
],
"type": "datetime"
},
{
"condition": "AND",
"filterType": "custom",
"key": "priority",
"operator": "=",
"type": "String",
"value": "error"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter1"
}
],
"limit": 5000,
"metrics": [],
"mode": "trend",
"orderBy": [],
"table": "log_docker_raw",
"timeField": "logdatetime",
"timeFieldType": "DateTime"
}
},
"queryType": "sql",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( visitParamExtractString(message, 'level') = 'ERROR' ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) AND ( message LIKE '%$filter2%' ) AND ( message LIKE '%$filter3%' ) GROUP BY time ORDER BY time ASC LIMIT 5000\n\n",
"refId": "A",
"selectedFormat": 4
} }
], ],
"title": "Total errors", "title": "Total errors",
@ -401,8 +340,9 @@
}, },
{ {
"datasource": { "datasource": {
"default": true,
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -452,106 +392,6 @@
"unit": "string" "unit": "string"
}, },
"overrides": [ "overrides": [
{
"matcher": {
"id": "byName",
"options": "logdatetime"
},
"properties": [
{
"id": "custom.width",
"value": 196
}
]
},
{
"matcher": {
"id": "byName",
"options": "host"
},
"properties": [
{
"id": "custom.width",
"value": 132
}
]
},
{
"matcher": {
"id": "byName",
"options": "program"
},
"properties": [
{
"id": "custom.width",
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "priority"
},
"properties": [
{
"id": "custom.width",
"value": 71
}
]
},
{
"matcher": {
"id": "byName",
"options": "time"
},
"properties": [
{
"id": "custom.width",
"value": 266
}
]
},
{
"matcher": {
"id": "byName",
"options": "level"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "handler"
},
"properties": [
{
"id": "custom.width",
"value": 217
}
]
},
{
"matcher": {
"id": "byName",
"options": "msg"
},
"properties": [
{
"id": "custom.width",
"value": 1307
},
{
"id": "custom.inspect",
"value": true
}
]
},
{ {
"matcher": { "matcher": {
"id": "byName", "id": "byName",
@ -560,7 +400,11 @@
"properties": [ "properties": [
{ {
"id": "custom.width", "id": "custom.width",
"value": 1466 "value": 658
},
{
"id": "custom.inspect",
"value": true
} }
] ]
}, },
@ -569,48 +413,6 @@
"id": "byName", "id": "byName",
"options": "Context" "options": "Context"
}, },
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
},
{
"id": "custom.inspect",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "connection"
},
"properties": [
{
"id": "custom.width",
"value": 220
}
]
},
{
"matcher": {
"id": "byName",
"options": "account"
},
"properties": [
{
"id": "custom.width",
"value": 223
}
]
},
{
"matcher": {
"id": "byName",
"options": "Raw"
},
"properties": [ "properties": [
{ {
"id": "custom.cellOptions", "id": "custom.cellOptions",
@ -624,32 +426,6 @@
} }
] ]
}, },
{
"matcher": {
"id": "byName",
"options": "Message"
},
"properties": [
{
"id": "custom.inspect",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "Priority"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
},
{ {
"matcher": { "matcher": {
"id": "byName", "id": "byName",
@ -661,6 +437,144 @@
"value": { "value": {
"type": "json-view" "type": "json-view"
} }
},
{
"id": "custom.width",
"value": 272
}
]
},
{
"matcher": {
"id": "byName",
"options": "Account"
},
"properties": [
{
"id": "custom.width",
"value": 158
}
]
},
{
"matcher": {
"id": "byName",
"options": "Context"
},
"properties": [
{
"id": "custom.inspect",
"value": true
},
{
"id": "custom.cellOptions",
"value": {
"type": "json-view"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "TS"
},
"properties": [
{
"id": "unit",
"value": "dateTimeAsIso"
},
{
"id": "custom.width",
"value": 176
}
]
},
{
"matcher": {
"id": "byName",
"options": "Stream ID"
},
"properties": [
{
"id": "custom.width",
"value": 234
}
]
},
{
"matcher": {
"id": "byName",
"options": "Priority"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-background"
}
},
{
"id": "mappings",
"value": [
{
"options": {
"DEBUG": {
"color": "green",
"index": 0,
"text": "Debug"
},
"ERROR": {
"color": "red",
"index": 2,
"text": "Error"
},
"INFO": {
"color": "blue",
"index": 1,
"text": "Info"
},
"WARN": {
"color": "yellow",
"index": 4,
"text": "Warning"
},
"WARNING": {
"color": "yellow",
"index": 3,
"text": "Warning"
}
},
"type": "value"
},
{
"options": {
"match": "empty",
"result": {
"color": "transparent",
"index": 5,
"text": "(not set)"
}
},
"type": "special"
}
]
},
{
"id": "custom.width",
"value": 118
}
]
},
{
"matcher": {
"id": "byName",
"options": "Connection"
},
"properties": [
{
"id": "custom.width",
"value": 250
} }
] ]
} }
@ -687,7 +601,7 @@
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": []
}, },
"pluginVersion": "10.2.1", "pluginVersion": "11.2.0",
"targets": [ "targets": [
{ {
"builderOptions": { "builderOptions": {
@ -739,7 +653,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"format": 1, "format": 1,
"meta": { "meta": {
@ -792,7 +706,7 @@
} }
}, },
"queryType": "sql", "queryType": "sql",
"rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", "rawSql": "SELECT \n visitParamExtractString(message, 'datetime') as t,\n visitParamExtractString(message, 'level_name') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'streamId') as stream_id,\n visitParamExtractString(message, 'message') as msg,\n replaceAll(visitParamExtractRaw(message, 'context'), '\\\\\\\\', '\\\\') as context\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
"refId": "A", "refId": "A",
"selectedFormat": 4 "selectedFormat": 4
} }
@ -803,14 +717,19 @@
"id": "organize", "id": "organize",
"options": { "options": {
"excludeByName": {}, "excludeByName": {},
"includeByName": {},
"indexByName": {}, "indexByName": {},
"renameByName": { "renameByName": {
"account": "Account", "account": "Account",
"connection": "Connection", "connection": "Connection",
"context": "Context",
"ctx": "Context",
"handler": "Handler", "handler": "Handler",
"level": "Priority", "level": "Priority",
"msg": "Message", "msg": "Message",
"raw": "Raw", "raw": "Raw",
"stream_id": "Stream ID",
"t": "TS",
"time": "TS" "time": "TS"
} }
} }
@ -819,7 +738,7 @@
"type": "table" "type": "table"
}, },
{ {
"collapsed": true, "collapsed": false,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
@ -827,11 +746,14 @@
"y": 37 "y": 37
}, },
"id": 1, "id": 1,
"panels": [ "panels": [],
"title": "Logs",
"type": "row"
},
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -850,7 +772,8 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green" "color": "green",
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -932,7 +855,7 @@
"h": 27, "h": 27,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 11 "y": 38
}, },
"id": 2, "id": 2,
"options": { "options": {
@ -948,7 +871,7 @@
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": []
}, },
"pluginVersion": "10.1.5", "pluginVersion": "11.2.0",
"targets": [ "targets": [
{ {
"builderOptions": { "builderOptions": {
@ -1001,7 +924,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"format": 1, "format": 1,
"meta": { "meta": {
@ -1021,24 +944,20 @@
"type": "table" "type": "table"
} }
], ],
"title": "Logs",
"type": "row"
}
],
"refresh": "", "refresh": "",
"schemaVersion": 38, "schemaVersion": 39,
"tags": [], "tags": [],
"templating": { "templating": {
"list": [ "list": [
{ {
"current": { "current": {
"selected": false, "selected": false,
"text": "telegram-mg_telegram-1", "text": "whatsapp-360-dialog-app-1",
"value": "telegram-mg_telegram-1" "value": "whatsapp-360-dialog-app-1"
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;",
"hide": 0, "hide": 0,
@ -1149,6 +1068,6 @@
"timezone": "", "timezone": "",
"title": "Logs + Structured via Query", "title": "Logs + Structured via Query",
"uid": "b445eb43-25d1-4112-b628-3b8acf037f10", "uid": "b445eb43-25d1-4112-b628-3b8acf037f10",
"version": 6, "version": 8,
"weekStart": "" "weekStart": ""
} }

36
docker-compose.yml
View File

@ -1,36 +0,0 @@
version: '3.8'
services:
grafana:
image: grafana/grafana:latest
volumes:
- grafana_data:/var/lib/grafana
ports:
- '3000:3000'
labels:
traefik.enable: "true"
traefik.http.routers.grafana.entrypoints: web
traefik.http.routers.grafana.rule: "Host(`grafana.test`)"
traefik.http.services.grafana.loadbalancer.server.port: "3000"
clickhouse:
image: docker.io/bitnami/clickhouse:23
environment:
- CLICKHOUSE_ADMIN_USER=${CLICKHOUSE_USER}
- CLICKHOUSE_ADMIN_PASSWORD=${CLICKHOUSE_PASSWORD}
ports:
- '${CLICKHOUSE_PORT}:8123'
volumes:
- clickhouse_data:/bitnami/clickhouse
vector:
image: timberio/vector:latest-alpine
ports:
- '8686:8686'
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes:
clickhouse_data:
driver: local
grafana_data:
driver: local

4
files/vector.yaml
View File

@ -1,6 +1,6 @@
api: api:
enabled: true enabled: true
address: '0.0.0.0:8686' address: 0.0.0.0:8686
sources: sources:
docker: docker:
type: docker_logs type: docker_logs
@ -34,7 +34,7 @@ sinks:
inputs: inputs:
- dedupe - dedupe
compression: gzip compression: gzip
endpoint: 'http://clickhouse:8123' endpoint: http://clickhouse:8123
auth: auth:
strategy: basic strategy: basic
user: default user: default