Neur0toxine/docker-logs-grafana-clickhouse
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Pavel 2024-05-21 14:56:35 +03:00
parent 261e79af54
commit a4ed616503
4 changed files with 238 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
dashboards/Logs + Structured via Query.json
View File

@ -18,7 +18,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 3,
"id": 1,
"links": [],
"liveNow": false,
"panels": [
@ -38,7 +38,7 @@
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"description": "",
"fieldConfig": {
@ -47,6 +47,7 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "series",
"axisLabel": "",
@ -174,7 +175,7 @@
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@ -187,7 +188,7 @@
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"fieldConfig": {
"defaults": {
@ -195,6 +196,7 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
@ -311,7 +313,7 @@
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"queryType": "builder",
"rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000",
@ -337,7 +339,7 @@
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"fieldConfig": {
"defaults": {
@ -622,7 +624,7 @@
"showHeader": true,
"sortBy": []
},
"pluginVersion": "10.1.5",
"pluginVersion": "10.4.1",
"targets": [
{
"builderOptions": {
@ -674,7 +676,7 @@
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"format": 1,
"meta": {
@ -727,7 +729,7 @@
}
},
"queryType": "sql",
"rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
"rawSql": "SELECT \n visitParamExtractString(message, 'datetime') as time,\n visitParamExtractString(message, 'level_name') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'message') as msg,\n mapFilter((k, v) -> k NOT IN ('datetime', 'level_name', 'connection', 'account', 'handler', 'message'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000",
"refId": "A",
"selectedFormat": 4
}
@ -754,7 +756,7 @@
"type": "table"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@ -762,11 +764,14 @@
"y": 37
},
"id": 1,
"panels": [
"panels": [],
"title": "Logs",
"type": "row"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"fieldConfig": {
"defaults": {
@ -867,7 +872,7 @@
"h": 27,
"w": 24,
"x": 0,
"y": 11
"y": 38
},
"id": 2,
"options": {
@ -883,7 +888,7 @@
"showHeader": true,
"sortBy": []
},
"pluginVersion": "10.1.5",
"pluginVersion": "10.4.1",
"targets": [
{
"builderOptions": {
@ -936,7 +941,7 @@
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"format": 1,
"meta": {
@ -956,13 +961,8 @@
"type": "table"
}
],
"title": "Logs",
"type": "row"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": [
@ -974,7 +974,7 @@
},
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "f70e773b-26c3-4b74-9a45-291fab27524b"
"uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4"
},
"definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;",
"hide": 0,
@ -1085,6 +1085,6 @@
"timezone": "",
"title": "Logs + Structured via Query",
"uid": "b445eb43-25d1-4112-b628-3b8acf037f10",
"version": 8,
"version": 4,
"weekStart": ""
}

2
docker-compose.yml
View File

@ -22,7 +22,7 @@ services:
- '8686:8686'
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- ./files/vector.toml:/etc/vector/vector.toml:ro
- ./files/vector.yaml:/etc/vector/vector.yaml:ro
volumes:
clickhouse_data:

38
files/vector.toml
View File

@ -1,38 +0,0 @@
[api]
enabled = true
address = "0.0.0.0:8686"
[sources.docker]
type = "docker_logs"
exclude_containers = [ "clickhouse", "grafana", "vector" ]
[transforms.ts_and_program]
type = "remap"
inputs = [ "docker" ]
source = '''
.logdatetime = to_unix_timestamp(now())
.program = .container_name
'''
[transforms.emptyprogram]
type = "filter"
inputs = [ "ts_and_program" ]
condition.type = "vrl"
condition.source = '''
.program != ""
'''
[transforms.dedupe]
type = "dedupe"
inputs = [ "emptyprogram" ]
[sinks.clickhouse]
type = "clickhouse"
inputs = [ "dedupe"]
compression = "gzip"
endpoint = "http://clickhouse:8123"
auth.strategy = "basic"
auth.user = "default"
auth.password = "default"
database = "default"
table = "log_docker_raw"

43
files/vector.yaml Normal file
View File

@ -0,0 +1,43 @@
api:
enabled: true
address: 0.0.0.0:8686
sources:
docker:
type: docker_logs
exclude_containers:
- clickhouse
- grafana
- vector
transforms:
ts_and_program:
type: remap
inputs:
- docker
source: |2
.logdatetime = to_unix_timestamp(now())
.program = .container_name
emptyprogram:
type: filter
inputs:
- ts_and_program
condition:
type: vrl
source: |2
.program != ""
dedupe:
type: dedupe
inputs:
- emptyprogram
sinks:
clickhouse:
type: clickhouse
inputs:
- dedupe
compression: gzip
endpoint: http://clickhouse:8123
auth:
strategy: basic
user: default
password: default
database: default
table: log_docker_raw