Neur0toxine/docker-logs-grafana-clickhouse
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Pavel 2024-05-21 14:56:35 +03:00
parent 261e79af54
commit a4ed616503
4 changed files with 238 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

388
dashboards/Logs + Structured via Query.json
View File

@ -18,7 +18,7 @@
"editable": true, "editable": true,
"fiscalYearStartMonth": 0, "fiscalYearStartMonth": 0,
"graphTooltip": 0, "graphTooltip": 0,
"id": 3, "id": 1,
"links": [], "links": [],
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
@ -38,7 +38,7 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"description": "", "description": "",
"fieldConfig": { "fieldConfig": {
@ -47,6 +47,7 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "series", "axisColorMode": "series",
"axisLabel": "", "axisLabel": "",
@ -174,7 +175,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"queryType": "builder", "queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@ -187,7 +188,7 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -195,6 +196,7 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
@ -311,7 +313,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"queryType": "builder", "queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@ -337,7 +339,7 @@
{ {
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
@ -622,7 +624,7 @@
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": []
}, },
"pluginVersion": "10.1.5", "pluginVersion": "10.4.1",
"targets": [ "targets": [
{ {
"builderOptions": { "builderOptions": {
@ -674,7 +676,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"format": 1, "format": 1,
"meta": { "meta": {
@ -727,7 +729,7 @@
} }
}, },
"queryType": "sql", "queryType": "sql",
"rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", "rawSql": "SELECT \n visitParamExtractString(message, 'datetime') as time,\n visitParamExtractString(message, 'level_name') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'message') as msg,\n mapFilter((k, v) -> k NOT IN ('datetime', 'level_name', 'connection', 'account', 'handler', 'message'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
"refId": "A", "refId": "A",
"selectedFormat": 4 "selectedFormat": 4
} }
@ -754,7 +756,7 @@
"type": "table" "type": "table"
}, },
{ {
"collapsed": true, "collapsed": false,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
@ -762,207 +764,205 @@
"y": 37 "y": 37
}, },
"id": 1, "id": 1,
"panels": [ "panels": [],
{ "title": "Logs",
"datasource": { "type": "row"
"type": "grafana-clickhouse-datasource", },
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" {
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
}, },
"fieldConfig": { "custom": {
"defaults": { "align": "auto",
"color": { "cellOptions": {
"mode": "thresholds" "type": "auto"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
}, },
"overrides": [ "inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ {
"matcher": { "color": "green"
"id": "byName",
"options": "logdatetime"
},
"properties": [
{
"id": "custom.width",
"value": 196
}
]
}, },
{ {
"matcher": { "color": "red",
"id": "byName", "value": 80
"options": "host" }
}, ]
"properties": [ }
{ },
"id": "custom.width", "overrides": [
"value": 132 {
} "matcher": {
] "id": "byName",
}, "options": "logdatetime"
},
"properties": [
{ {
"matcher": { "id": "custom.width",
"id": "byName", "value": 196
"options": "program"
},
"properties": [
{
"id": "custom.width",
"value": 210
}
]
},
{
"matcher": {
"id": "byName",
"options": "priority"
},
"properties": [
{
"id": "custom.width",
"value": 71
}
]
},
{
"matcher": {
"id": "byName",
"options": "message"
},
"properties": [
{
"id": "custom.cellOptions",
"value": {
"type": "json-view"
}
},
{
"id": "custom.inspect",
"value": true
}
]
} }
] ]
}, },
"gridPos": { {
"h": 27, "matcher": {
"w": 24, "id": "byName",
"x": 0, "options": "host"
"y": 11
},
"id": 2,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
}, },
"showHeader": true, "properties": [
"sortBy": [] {
"id": "custom.width",
"value": 132
}
]
}, },
"pluginVersion": "10.1.5", {
"targets": [ "matcher": {
{ "id": "byName",
"builderOptions": { "options": "program"
"database": "default", },
"fields": [ "properties": [
"host", {
"program", "id": "custom.width",
"logdatetime", "value": 210
"priority", }
"message" ]
], },
"filters": [ {
{ "matcher": {
"condition": "AND", "id": "byName",
"filterType": "custom", "options": "priority"
"key": "logdatetime", },
"operator": "WITH IN DASHBOARD TIME RANGE", "properties": [
"type": "DateTime", {
"value": "TODAY" "id": "custom.width",
}, "value": 71
{ }
"condition": "AND", ]
"filterType": "custom", },
"key": "program", {
"operator": "=", "matcher": {
"type": "String", "id": "byName",
"value": "${program:singlequote}" "options": "message"
}, },
{ "properties": [
"condition": "AND", {
"filterType": "custom", "id": "custom.cellOptions",
"key": "message", "value": {
"operator": "LIKE", "type": "json-view"
"type": "String",
"value": "$filter1"
}
],
"limit": 5000,
"metrics": [],
"mode": "list",
"orderBy": [
{
"dir": "DESC",
"name": "logdatetime"
}
],
"table": "log_docker_raw",
"timeField": "logdatetime",
"timeFieldType": "DateTime"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
},
"format": 1,
"meta": {
"builderOptions": {
"fields": [],
"limit": 100,
"mode": "list"
} }
}, },
"queryType": "builder", {
"rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", "id": "custom.inspect",
"refId": "A", "value": true
"selectedFormat": 4 }
} ]
}
]
},
"gridPos": {
"h": 27,
"w": 24,
"x": 0,
"y": 38
},
"id": 2,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
], ],
"title": "Panel Title", "show": false
"type": "table" },
"showHeader": true,
"sortBy": []
},
"pluginVersion": "10.4.1",
"targets": [
{
"builderOptions": {
"database": "default",
"fields": [
"host",
"program",
"logdatetime",
"priority",
"message"
],
"filters": [
{
"condition": "AND",
"filterType": "custom",
"key": "logdatetime",
"operator": "WITH IN DASHBOARD TIME RANGE",
"type": "DateTime",
"value": "TODAY"
},
{
"condition": "AND",
"filterType": "custom",
"key": "program",
"operator": "=",
"type": "String",
"value": "${program:singlequote}"
},
{
"condition": "AND",
"filterType": "custom",
"key": "message",
"operator": "LIKE",
"type": "String",
"value": "$filter1"
}
],
"limit": 5000,
"metrics": [],
"mode": "list",
"orderBy": [
{
"dir": "DESC",
"name": "logdatetime"
}
],
"table": "log_docker_raw",
"timeField": "logdatetime",
"timeFieldType": "DateTime"
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"format": 1,
"meta": {
"builderOptions": {
"fields": [],
"limit": 100,
"mode": "list"
}
},
"queryType": "builder",
"rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000",
"refId": "A",
"selectedFormat": 4
} }
], ],
"title": "Logs", "title": "Panel Title",
"type": "row" "type": "table"
} }
], ],
"refresh": "", "refresh": "",
"schemaVersion": 38, "schemaVersion": 39,
"style": "dark",
"tags": [], "tags": [],
"templating": { "templating": {
"list": [ "list": [
@ -974,7 +974,7 @@
}, },
"datasource": { "datasource": {
"type": "grafana-clickhouse-datasource", "type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b" "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
}, },
"definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;",
"hide": 0, "hide": 0,
@ -1085,6 +1085,6 @@
"timezone": "", "timezone": "",
"title": "Logs + Structured via Query", "title": "Logs + Structured via Query",
"uid": "b445eb43-25d1-4112-b628-3b8acf037f10", "uid": "b445eb43-25d1-4112-b628-3b8acf037f10",
"version": 8, "version": 4,
"weekStart": "" "weekStart": ""
} }

2
docker-compose.yml
View File

@ -22,7 +22,7 @@ services:
- '8686:8686' - '8686:8686'
volumes: volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro - /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.toml:/etc/vector/vector.toml:ro - ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes: volumes:
clickhouse_data: clickhouse_data:

38
files/vector.toml
View File

@ -1,38 +0,0 @@
[api]
enabled = true
address = "0.0.0.0:8686"
[sources.docker]
type = "docker_logs"
exclude_containers = [ "clickhouse", "grafana", "vector" ]
[transforms.ts_and_program]
type = "remap"
inputs = [ "docker" ]
source = '''
.logdatetime = to_unix_timestamp(now())
.program = .container_name
'''
[transforms.emptyprogram]
type = "filter"
inputs = [ "ts_and_program" ]
condition.type = "vrl"
condition.source = '''
.program != ""
'''
[transforms.dedupe]
type = "dedupe"
inputs = [ "emptyprogram" ]
[sinks.clickhouse]
type = "clickhouse"
inputs = [ "dedupe"]
compression = "gzip"
endpoint = "http://clickhouse:8123"
auth.strategy = "basic"
auth.user = "default"
auth.password = "default"
database = "default"
table = "log_docker_raw"

43
files/vector.yaml Normal file
View File

@ -0,0 +1,43 @@
api:
enabled: true
address: 0.0.0.0:8686
sources:
docker:
type: docker_logs
exclude_containers:
- clickhouse
- grafana
- vector
transforms:
ts_and_program:
type: remap
inputs:
- docker
source: |2
.logdatetime = to_unix_timestamp(now())
.program = .container_name
emptyprogram:
type: filter
inputs:
- ts_and_program
condition:
type: vrl
source: |2
.program != ""
dedupe:
type: dedupe
inputs:
- emptyprogram
sinks:
clickhouse:
type: clickhouse
inputs:
- dedupe
compression: gzip
endpoint: http://clickhouse:8123
auth:
strategy: basic
user: default
password: default
database: default
table: log_docker_raw