Neur0toxine/docker-logs-grafana-clickhouse
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: Neur0toxine:f3382f71f312e6e6981d71357e3d16fbabdeaffe
Branches Tags
Neur0toxine:master
..
compare: Neur0toxine:3a8a0fb79830bac9077656b29df3c50ef2e26298
Branches Tags
Neur0toxine:master

No commits in common. "f3382f71f312e6e6981d71357e3d16fbabdeaffe" and "3a8a0fb79830bac9077656b29df3c50ef2e26298" have entirely different histories.
f3382f71f3 ... 3a8a0fb798

5 changed files with 550 additions and 486 deletions
Show Stats Expand all files Collapse all files

8
README.md
View File

@ -1,6 +1,6 @@
# Grafana Clickhouse Docker Log Aggregation # Grafana Clickhouse Docker Log Aggregation
This demo project can aggregate logs from Docker containers and store them in Clickhouse. In the `dashboards` directory you can find dashboard examples for specific log format. `Logs + Structured via Query` is recommended because the other one is deprecated now. This demo project can aggregate logs from Docker containers and store them in Clickhouse. In the `dashboards` directory you can find dashboard examples for specific log format. `Logs + Structured via Query` is recommended.
**Note**: dashboards are tuned for structured logging (JSON log records). **Note**: dashboards are tuned for structured logging (JSON log records).
@ -8,8 +8,8 @@ This demo project can aggregate logs from Docker containers and store them in Cl
Obviously, you'll need Docker: Obviously, you'll need Docker:
1. `make migrate` (use `clickhouse-client` inside container and execute migration SQL directly if migrator doesn't work). 1. `make all`
2. Navigate to `http://grafana.test` or `http://localhost:3000` (if you did not configure Traefik locally), use `admin:admin` as credentials. 2. Navigate to `http://grafana.test` or `http://localhost:3000`, use `admin:admin` as credentials.
3. Import `dashboards/Logs + Structured via Query.json` dashboard. 3. Create your own dashboard or import those I provided (`dashboards/` directory). Replace data source UUID with correct one if needed.
4. Run any Docker project with structured JSON logs. 4. Run any Docker project with structured JSON logs.
5. Done. 5. Done.

53
compose.yml
View File

@ -1,53 +0,0 @@
services:
grafana:
image: grafana/grafana:latest
volumes:
- grafana_data:/var/lib/grafana
depends_on:
- clickhouse
ports:
- 3000:3000
labels:
traefik.enable: "true"
traefik.http.routers.grafana.entrypoints: web
traefik.http.routers.grafana.rule: 'Host(`grafana.test`)'
traefik.http.services.grafana.loadbalancer.server.port: 3000
clickhouse:
image: clickhouse/clickhouse-server:24
environment:
- CLICKHOUSE_DB=default
- CLICKHOUSE_USER=default
- CLICKHOUSE_PASSWORD=default
- CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
ports:
- '8123:8123'
ulimits:
nofile:
soft: 262144
hard: 262144
cap_add:
- NET_ADMIN
- SYS_NICE
- IPC_LOCK
healthcheck:
test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
interval: 10s
timeout: 5s
retries: 5
volumes:
- clickhouse_data:/var/lib/clickhouse
vector:
image: timberio/vector:latest-alpine
ports:
- '8686:8686'
depends_on:
- clickhouse
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes:
clickhouse_data:
driver: local
grafana_data:
driver: local

451
dashboards/Logs + Structured via Query.json
View File

@ -38,7 +38,7 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"description": "", "description": "",
"fieldConfig": { "fieldConfig": {
@ -53,7 +53,6 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 0, "fillOpacity": 0,
"gradientMode": "hue", "gradientMode": "hue",
@ -158,6 +157,22 @@
"operator": "LIKE", "operator": "LIKE",
"type": "String", "type": "String",
"value": "$filter1" "value": "$filter1"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter2"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter3"
} }
], ],
"limit": 5000, "limit": 5000,
@ -176,10 +191,10 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"queryType": "builder", "queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) AND ( message LIKE '%$filter2%' ) AND ( message LIKE '%$filter3%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
"refId": "A" "refId": "A"
} }
], ],
@ -189,12 +204,13 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {
"mode": "palette-classic" "fixedColor": "red",
"mode": "fixed"
}, },
"custom": { "custom": {
"axisBorderShow": false, "axisBorderShow": false,
@ -203,7 +219,6 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 0, "fillOpacity": 0,
"gradientMode": "none", "gradientMode": "none",
@ -233,13 +248,9 @@
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{
"color": "green",
"value": null
},
{ {
"color": "red", "color": "red",
"value": 80 "value": null
} }
] ]
} }
@ -315,11 +326,61 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"queryType": "builder", "format": 1,
"rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", "meta": {
"refId": "A" "builderOptions": {
"database": "default",
"fields": [
"priority"
],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "logdatetime",
"operator": "WITH IN DASHBOARD TIME RANGE",
"restrictToFields": [
{
"label": "logdatetime",
"name": "logdatetime",
"picklistValues": [],
"type": "DateTime"
}
],
"type": "datetime"
},
{
"condition": "AND",
"filterType": "custom",
"key": "priority",
"operator": "=",
"type": "String",
"value": "error"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter1"
}
],
"limit": 5000,
"metrics": [],
"mode": "trend",
"orderBy": [],
"table": "log_docker_raw",
"timeField": "logdatetime",
"timeFieldType": "DateTime"
}
},
"queryType": "sql",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( visitParamExtractString(message, 'level') = 'ERROR' ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) AND ( message LIKE '%$filter2%' ) AND ( message LIKE '%$filter3%' ) GROUP BY time ORDER BY time ASC LIMIT 5000\n\n",
"refId": "A",
"selectedFormat": 4
} }
], ],
"title": "Total errors", "title": "Total errors",
@ -340,9 +401,8 @@
}, },
{ {
"datasource": { "datasource": {
"default": true,
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -392,6 +452,106 @@
"unit": "string" "unit": "string"
}, },
"overrides": [ "overrides": [
{
"matcher": {
"id": "byName",
"options": "logdatetime"
},
"properties": [
{
"id": "custom.width",
"value": 196
}
]
},
{
"matcher": {
"id": "byName",
"options": "host"
},
"properties": [
{
"id": "custom.width",
"value": 132
}
]
},
{
"matcher": {
"id": "byName",
"options": "program"
},
"properties": [
{
"id": "custom.width",
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "priority"
},
"properties": [
{
"id": "custom.width",
"value": 71
}
]
},
{
"matcher": {
"id": "byName",
"options": "time"
},
"properties": [
{
"id": "custom.width",
"value": 266
}
]
},
{
"matcher": {
"id": "byName",
"options": "level"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "handler"
},
"properties": [
{
"id": "custom.width",
"value": 217
}
]
},
{
"matcher": {
"id": "byName",
"options": "msg"
},
"properties": [
{
"id": "custom.width",
"value": 1307
},
{
"id": "custom.inspect",
"value": true
}
]
},
{ {
"matcher": { "matcher": {
"id": "byName", "id": "byName",
@ -400,11 +560,7 @@
"properties": [ "properties": [
{ {
"id": "custom.width", "id": "custom.width",
"value": 658 "value": 1466
},
{
"id": "custom.inspect",
"value": true
} }
] ]
}, },
@ -413,6 +569,48 @@
"id": "byName", "id": "byName",
"options": "Context" "options": "Context"
}, },
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
},
{
"id": "custom.inspect",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "connection"
},
"properties": [
{
"id": "custom.width",
"value": 220
}
]
},
{
"matcher": {
"id": "byName",
"options": "account"
},
"properties": [
{
"id": "custom.width",
"value": 223
}
]
},
{
"matcher": {
"id": "byName",
"options": "Raw"
},
"properties": [ "properties": [
{ {
"id": "custom.cellOptions", "id": "custom.cellOptions",
@ -426,6 +624,32 @@
} }
] ]
}, },
{
"matcher": {
"id": "byName",
"options": "Message"
},
"properties": [
{
"id": "custom.inspect",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "Priority"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-text"
}
}
]
},
{ {
"matcher": { "matcher": {
"id": "byName", "id": "byName",
@ -437,144 +661,6 @@
"value": { "value": {
"type": "json-view" "type": "json-view"
} }
},
{
"id": "custom.width",
"value": 272
}
]
},
{
"matcher": {
"id": "byName",
"options": "Account"
},
"properties": [
{
"id": "custom.width",
"value": 158
}
]
},
{
"matcher": {
"id": "byName",
"options": "Context"
},
"properties": [
{
"id": "custom.inspect",
"value": true
},
{
"id": "custom.cellOptions",
"value": {
"type": "json-view"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "TS"
},
"properties": [
{
"id": "unit",
"value": "dateTimeAsIso"
},
{
"id": "custom.width",
"value": 176
}
]
},
{
"matcher": {
"id": "byName",
"options": "Stream ID"
},
"properties": [
{
"id": "custom.width",
"value": 234
}
]
},
{
"matcher": {
"id": "byName",
"options": "Priority"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "color-background"
}
},
{
"id": "mappings",
"value": [
{
"options": {
"DEBUG": {
"color": "green",
"index": 0,
"text": "Debug"
},
"ERROR": {
"color": "red",
"index": 2,
"text": "Error"
},
"INFO": {
"color": "blue",
"index": 1,
"text": "Info"
},
"WARN": {
"color": "yellow",
"index": 4,
"text": "Warning"
},
"WARNING": {
"color": "yellow",
"index": 3,
"text": "Warning"
}
},
"type": "value"
},
{
"options": {
"match": "empty",
"result": {
"color": "transparent",
"index": 5,
"text": "(not set)"
}
},
"type": "special"
}
]
},
{
"id": "custom.width",
"value": 118
}
]
},
{
"matcher": {
"id": "byName",
"options": "Connection"
},
"properties": [
{
"id": "custom.width",
"value": 250
} }
] ]
} }
@ -601,7 +687,7 @@
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": []
}, },
"pluginVersion": "11.2.0", "pluginVersion": "10.2.1",
"targets": [ "targets": [
{ {
"builderOptions": { "builderOptions": {
@ -653,7 +739,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"format": 1, "format": 1,
"meta": { "meta": {
@ -706,7 +792,7 @@
} }
}, },
"queryType": "sql", "queryType": "sql",
"rawSql": "SELECT \n visitParamExtractString(message, 'datetime') as t,\n visitParamExtractString(message, 'level_name') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'streamId') as stream_id,\n visitParamExtractString(message, 'message') as msg,\n replaceAll(visitParamExtractRaw(message, 'context'), '\\\\\\\\', '\\\\') as context\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", "rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
"refId": "A", "refId": "A",
"selectedFormat": 4 "selectedFormat": 4
} }
@ -717,19 +803,14 @@
"id": "organize", "id": "organize",
"options": { "options": {
"excludeByName": {}, "excludeByName": {},
"includeByName": {},
"indexByName": {}, "indexByName": {},
"renameByName": { "renameByName": {
"account": "Account", "account": "Account",
"connection": "Connection", "connection": "Connection",
"context": "Context",
"ctx": "Context",
"handler": "Handler", "handler": "Handler",
"level": "Priority", "level": "Priority",
"msg": "Message", "msg": "Message",
"raw": "Raw", "raw": "Raw",
"stream_id": "Stream ID",
"t": "TS",
"time": "TS" "time": "TS"
} }
} }
@ -738,7 +819,7 @@
"type": "table" "type": "table"
}, },
{ {
"collapsed": false, "collapsed": true,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
@ -746,14 +827,11 @@
"y": 37 "y": 37
}, },
"id": 1, "id": 1,
"panels": [], "panels": [
"title": "Logs",
"type": "row"
},
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -772,8 +850,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -855,7 +932,7 @@
"h": 27, "h": 27,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 38 "y": 11
}, },
"id": 2, "id": 2,
"options": { "options": {
@ -871,7 +948,7 @@
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": []
}, },
"pluginVersion": "11.2.0", "pluginVersion": "10.1.5",
"targets": [ "targets": [
{ {
"builderOptions": { "builderOptions": {
@ -924,7 +1001,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"format": 1, "format": 1,
"meta": { "meta": {
@ -944,20 +1021,24 @@
"type": "table" "type": "table"
} }
], ],
"title": "Logs",
"type": "row"
}
],
"refresh": "", "refresh": "",
"schemaVersion": 39, "schemaVersion": 38,
"tags": [], "tags": [],
"templating": { "templating": {
"list": [ "list": [
{ {
"current": { "current": {
"selected": false, "selected": false,
"text": "whatsapp-360-dialog-app-1", "text": "telegram-mg_telegram-1",
"value": "whatsapp-360-dialog-app-1" "value": "telegram-mg_telegram-1"
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" "uid": "ef7b4f5b-7011-411b-88c2-4b2681b8ddbe"
}, },
"definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;",
"hide": 0, "hide": 0,
@ -1068,6 +1149,6 @@
"timezone": "", "timezone": "",
"title": "Logs + Structured via Query", "title": "Logs + Structured via Query",
"uid": "b445eb43-25d1-4112-b628-3b8acf037f10", "uid": "b445eb43-25d1-4112-b628-3b8acf037f10",
"version": 8, "version": 6,
"weekStart": "" "weekStart": ""
} }

36
docker-compose.yml Normal file
View File

@ -0,0 +1,36 @@
version: '3.8'
services:
grafana:
image: grafana/grafana:latest
volumes:
- grafana_data:/var/lib/grafana
ports:
- '3000:3000'
labels:
traefik.enable: "true"
traefik.http.routers.grafana.entrypoints: web
traefik.http.routers.grafana.rule: "Host(`grafana.test`)"
traefik.http.services.grafana.loadbalancer.server.port: "3000"
clickhouse:
image: docker.io/bitnami/clickhouse:23
environment:
- CLICKHOUSE_ADMIN_USER=${CLICKHOUSE_USER}
- CLICKHOUSE_ADMIN_PASSWORD=${CLICKHOUSE_PASSWORD}
ports:
- '${CLICKHOUSE_PORT}:8123'
volumes:
- clickhouse_data:/bitnami/clickhouse
vector:
image: timberio/vector:latest-alpine
ports:
- '8686:8686'
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes:
clickhouse_data:
driver: local
grafana_data:
driver: local

4
files/vector.yaml
View File

@ -1,6 +1,6 @@
api: api:
enabled: true enabled: true
address: 0.0.0.0:8686 address: '0.0.0.0:8686'
sources: sources:
docker: docker:
type: docker_logs type: docker_logs
@ -34,7 +34,7 @@ sinks:
inputs: inputs:
- dedupe - dedupe
compression: gzip compression: gzip
endpoint: http://clickhouse:8123 endpoint: 'http://clickhouse:8123'
auth: auth:
strategy: basic strategy: basic
user: default user: default