From a4ed6165039a797a1e2cbbe45675425ad0c0b6e3 Mon Sep 17 00:00:00 2001 From: Neur0toxine Date: Tue, 21 May 2024 14:56:35 +0300 Subject: [PATCH] update --- dashboards/Logs + Structured via Query.json | 388 ++++++++++---------- docker-compose.yml | 2 +- files/vector.toml | 38 -- files/vector.yaml | 43 +++ 4 files changed, 238 insertions(+), 233 deletions(-) delete mode 100644 files/vector.toml create mode 100644 files/vector.yaml diff --git a/dashboards/Logs + Structured via Query.json b/dashboards/Logs + Structured via Query.json index 8479a28..cb980ce 100644 --- a/dashboards/Logs + Structured via Query.json +++ b/dashboards/Logs + Structured via Query.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 1, "links": [], "liveNow": false, "panels": [ @@ -38,7 +38,7 @@ { "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "description": "", "fieldConfig": { @@ -47,6 +47,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "series", "axisLabel": "", @@ -174,7 +175,7 @@ }, "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "queryType": "builder", "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", @@ -187,7 +188,7 @@ { "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "fieldConfig": { "defaults": { @@ -195,6 +196,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -311,7 +313,7 @@ }, "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "queryType": "builder", "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", @@ -337,7 +339,7 @@ { "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "fieldConfig": { "defaults": { @@ -622,7 +624,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "10.1.5", + "pluginVersion": "10.4.1", "targets": [ { "builderOptions": { @@ -674,7 +676,7 @@ }, "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "format": 1, "meta": { @@ -727,7 +729,7 @@ } }, "queryType": "sql", - "rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", + "rawSql": "SELECT \n visitParamExtractString(message, 'datetime') as time,\n visitParamExtractString(message, 'level_name') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'message') as msg,\n mapFilter((k, v) -> k NOT IN ('datetime', 'level_name', 'connection', 'account', 'handler', 'message'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", "refId": "A", "selectedFormat": 4 } @@ -754,7 +756,7 @@ "type": "table" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -762,207 +764,205 @@ "y": 37 }, "id": 1, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "overrides": [ + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "logdatetime" - }, - "properties": [ - { - "id": "custom.width", - "value": 196 - } - ] + "color": "green" }, { - "matcher": { - "id": "byName", - "options": "host" - }, - "properties": [ - { - "id": "custom.width", - "value": 132 - } - ] - }, + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ { - "matcher": { - "id": "byName", - "options": "program" - }, - "properties": [ - { - "id": "custom.width", - "value": 210 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "priority" - }, - "properties": [ - { - "id": "custom.width", - "value": 71 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "message" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "json-view" - } - }, - { - "id": "custom.inspect", - "value": true - } - ] + "id": "custom.width", + "value": 196 } ] }, - "gridPos": { - "h": 27, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 2, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + { + "matcher": { + "id": "byName", + "options": "host" }, - "showHeader": true, - "sortBy": [] + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] }, - "pluginVersion": "10.1.5", - "targets": [ - { - "builderOptions": { - "database": "default", - "fields": [ - "host", - "program", - "logdatetime", - "priority", - "message" - ], - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "logdatetime", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "DateTime", - "value": "TODAY" - }, - { - "condition": "AND", - "filterType": "custom", - "key": "program", - "operator": "=", - "type": "String", - "value": "${program:singlequote}" - }, - { - "condition": "AND", - "filterType": "custom", - "key": "message", - "operator": "LIKE", - "type": "String", - "value": "$filter1" - } - ], - "limit": 5000, - "metrics": [], - "mode": "list", - "orderBy": [ - { - "dir": "DESC", - "name": "logdatetime" - } - ], - "table": "log_docker_raw", - "timeField": "logdatetime", - "timeFieldType": "DateTime" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" - }, - "format": 1, - "meta": { - "builderOptions": { - "fields": [], - "limit": 100, - "mode": "list" + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "message" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" } }, - "queryType": "builder", - "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", - "refId": "A", - "selectedFormat": 4 - } + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" ], - "title": "Panel Title", - "type": "table" + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "priority", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "builder", + "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 } ], - "title": "Logs", - "type": "row" + "title": "Panel Title", + "type": "table" } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [ @@ -974,7 +974,7 @@ }, "datasource": { "type": "grafana-clickhouse-datasource", - "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + "uid": "c029a1fb-9305-40f9-ad75-bd648c3138b4" }, "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", "hide": 0, @@ -1085,6 +1085,6 @@ "timezone": "", "title": "Logs + Structured via Query", "uid": "b445eb43-25d1-4112-b628-3b8acf037f10", - "version": 8, + "version": 4, "weekStart": "" } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5131ab6..b1a6dfe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: - '8686:8686' volumes: - /var/run/docker.sock:/var/run/docker.sock:ro - - ./files/vector.toml:/etc/vector/vector.toml:ro + - ./files/vector.yaml:/etc/vector/vector.yaml:ro volumes: clickhouse_data: diff --git a/files/vector.toml b/files/vector.toml deleted file mode 100644 index a53054a..0000000 --- a/files/vector.toml +++ /dev/null @@ -1,38 +0,0 @@ -[api] -enabled = true -address = "0.0.0.0:8686" - -[sources.docker] -type = "docker_logs" -exclude_containers = [ "clickhouse", "grafana", "vector" ] - -[transforms.ts_and_program] -type = "remap" -inputs = [ "docker" ] -source = ''' - .logdatetime = to_unix_timestamp(now()) - .program = .container_name -''' - -[transforms.emptyprogram] -type = "filter" -inputs = [ "ts_and_program" ] -condition.type = "vrl" -condition.source = ''' - .program != "" -''' - -[transforms.dedupe] -type = "dedupe" -inputs = [ "emptyprogram" ] - -[sinks.clickhouse] -type = "clickhouse" -inputs = [ "dedupe"] -compression = "gzip" -endpoint = "http://clickhouse:8123" -auth.strategy = "basic" -auth.user = "default" -auth.password = "default" -database = "default" -table = "log_docker_raw" \ No newline at end of file diff --git a/files/vector.yaml b/files/vector.yaml new file mode 100644 index 0000000..da33765 --- /dev/null +++ b/files/vector.yaml @@ -0,0 +1,43 @@ +api: + enabled: true + address: 0.0.0.0:8686 +sources: + docker: + type: docker_logs + exclude_containers: + - clickhouse + - grafana + - vector +transforms: + ts_and_program: + type: remap + inputs: + - docker + source: |2 + .logdatetime = to_unix_timestamp(now()) + .program = .container_name + emptyprogram: + type: filter + inputs: + - ts_and_program + condition: + type: vrl + source: |2 + .program != "" + dedupe: + type: dedupe + inputs: + - emptyprogram +sinks: + clickhouse: + type: clickhouse + inputs: + - dedupe + compression: gzip + endpoint: http://clickhouse:8123 + auth: + strategy: basic + user: default + password: default + database: default + table: log_docker_raw