commit c71313d367c9ff77c3625452dcac9e6ae9e20ce6 Author: Neur0toxine Date: Tue Oct 24 15:46:25 2023 +0300 initial commit diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c57a52c --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +SHELL = /bin/bash -o pipefail + +ROOT_DIR=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +MIGRATIONS_DIR=$(SRC_DIR)/files/migrations + +migrate: + @docker-compose up -d + @docker run -v $(MIGRATIONS_DIR):/migrations --network host migrate/migrate -path=/migrations/ -database "clickhouse://localhost:8123?username=default&password=default&database=default" up all + @docker-compose stop + +up: + @docker-compose up diff --git a/dashboards/Logs + Structured via Query.json b/dashboards/Logs + Structured via Query.json new file mode 100644 index 0000000..8479a28 --- /dev/null +++ b/dashboards/Logs + Structured via Query.json @@ -0,0 +1,1090 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "Total & Errors", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "recps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "Europe/Moscow" + ], + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority", + "logdatetime" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [ + { + "aggregation": "count", + "alias": "", + "field": "" + } + ], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "priority", + "operator": "=", + "type": "String", + "value": "error" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "title": "Structured logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "displayName": "Priority", + "mappings": [ + { + "options": { + "DEBUG": { + "color": "dark-green", + "index": 0 + }, + "ERROR": { + "color": "dark-red", + "index": 3 + }, + "INFO": { + "color": "dark-blue", + "index": 1 + }, + "WARN": { + "color": "dark-yellow", + "index": 2 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "string" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time" + }, + "properties": [ + { + "id": "custom.width", + "value": 266 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "level" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "handler" + }, + "properties": [ + { + "id": "custom.width", + "value": 217 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "msg" + }, + "properties": [ + { + "id": "custom.width", + "value": 1307 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Message" + }, + "properties": [ + { + "id": "custom.width", + "value": 1466 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Context" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "connection" + }, + "properties": [ + { + "id": "custom.width", + "value": 220 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "account" + }, + "properties": [ + { + "id": "custom.width", + "value": 223 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Raw" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Message" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Priority" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Handler" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "format": 1, + "meta": { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + } + }, + "queryType": "sql", + "rawSql": "SELECT \n visitParamExtractString(message, 'time') as time,\n visitParamExtractString(message, 'level') as level,\n visitParamExtractString(message, 'connection') as connection,\n visitParamExtractString(message, 'account') as account,\n visitParamExtractString(message, 'handler') as handler,\n visitParamExtractString(message, 'msg') as msg,\n mapFilter((k, v) -> k NOT IN ('time', 'level', 'connection', 'account', 'handler', 'msg'), JSONExtract(message, 'Map(String, String)')) as raw\nFROM \"default\".\"log_docker_raw\"\nWHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime )\n AND IF('Any' IN (${priority}), 1 = 1, level IN (${priority}))\n AND ( program = ${program:singlequote} )\n AND ( message LIKE '%$filter1%' )\n AND ( message LIKE '%$filter2%' )\n AND ( message LIKE '%$filter3%' )\n AND ( msg <> '' )\nORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Structured logs output", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "account": "Account", + "connection": "Connection", + "handler": "Handler", + "level": "Priority", + "msg": "Message", + "raw": "Raw", + "time": "TS" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 1, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "message" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "priority", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "builder", + "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Panel Title", + "type": "table" + } + ], + "title": "Logs", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "telegram-mg_telegram-1", + "value": "telegram-mg_telegram-1" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "hide": 0, + "includeAll": false, + "label": "Program", + "multi": false, + "name": "program", + "options": [], + "query": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "Any" + ], + "value": [ + "Any" + ] + }, + "hide": 0, + "includeAll": false, + "label": "Priority", + "multi": true, + "name": "priority", + "options": [ + { + "selected": true, + "text": "Any", + "value": "Any" + }, + { + "selected": false, + "text": "DEBUG", + "value": "DEBUG" + }, + { + "selected": false, + "text": "INFO", + "value": "INFO" + }, + { + "selected": false, + "text": "WARN", + "value": "WARN" + }, + { + "selected": false, + "text": "ERROR", + "value": "ERROR" + } + ], + "query": "Any,DEBUG,INFO,WARN,ERROR", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter #1", + "name": "filter1", + "options": [ + { + "selected": true, + "text": "*", + "value": "*" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "name": "filter2", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "name": "filter3", + "options": [], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Logs + Structured via Query", + "uid": "b445eb43-25d1-4112-b628-3b8acf037f10", + "version": 8, + "weekStart": "" +} \ No newline at end of file diff --git a/dashboards/Logs + Structured via transforms.json b/dashboards/Logs + Structured via transforms.json new file mode 100644 index 0000000..22a769c --- /dev/null +++ b/dashboards/Logs + Structured via transforms.json @@ -0,0 +1,1009 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "Total & Errors", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "recps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "Europe/Moscow" + ], + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority", + "logdatetime" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [ + { + "aggregation": "count", + "alias": "", + "field": "" + } + ], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time, count() FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "priority" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "logdatetime", + "name": "logdatetime", + "picklistValues": [], + "type": "DateTime" + } + ], + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "priority", + "operator": "=", + "type": "String", + "value": "error" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "trend", + "orderBy": [], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "queryType": "builder", + "rawSql": "SELECT $__timeInterval(logdatetime) as time FROM \"default\".\"log_docker_raw\" WHERE $__timeFilter(logdatetime) AND ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( priority = 'error' ) AND ( message LIKE '%$filter1%' ) GROUP BY time ORDER BY time ASC LIMIT 5000", + "refId": "A" + } + ], + "title": "Total errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "title": "Structured logs", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time" + }, + "properties": [ + { + "id": "custom.width", + "value": 266 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "level" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "handler" + }, + "properties": [ + { + "id": "custom.width", + "value": 217 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "msg" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Message" + }, + "properties": [ + { + "id": "custom.width", + "value": 1610 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Context" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "builder", + "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Structured logs output", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "keepTime": false, + "replace": true, + "source": "message" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Context", + "mode": "reduceRow", + "reduce": { + "include": [ + "startTime", + "endTime", + "latency", + "remoteAddress", + "method", + "path", + "bodySize", + "messageId", + "body", + "userId", + "error", + "updateId", + "statusCode", + "handlerCount", + "minRequests", + "counterId" + ], + "reducer": "uniqueValues" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "body": true, + "bodySize": true, + "counterId": true, + "endTime": true, + "error": true, + "handlerCount": true, + "latency": true, + "messageId": true, + "method": true, + "minRequests": true, + "path": true, + "remoteAddress": true, + "startTime": true, + "statusCode": true, + "updateId": true, + "user": true, + "userId": true + }, + "indexByName": { + "Context": 23, + "account": 3, + "body": 14, + "bodySize": 12, + "connection": 2, + "counterId": 21, + "endTime": 7, + "error": 17, + "handler": 4, + "handlerCount": 20, + "latency": 8, + "level": 1, + "messageId": 13, + "method": 10, + "minRequests": 22, + "msg": 5, + "path": 11, + "remoteAddress": 9, + "startTime": 6, + "statusCode": 19, + "time": 0, + "updateId": 18, + "user": 15, + "userId": 16 + }, + "renameByName": { + "account": "Account", + "body": "Body", + "connection": "Connection", + "error": "Error", + "handler": "Handler", + "handlerCount": "Handler Count", + "level": "Priority", + "method": "Method", + "msg": "Message", + "path": "Path", + "statusCode": "Status Code", + "time": "TS" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 1, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "logdatetime" + }, + "properties": [ + { + "id": "custom.width", + "value": 196 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "host" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "program" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 71 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "message" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "json-view" + } + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 27, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "builderOptions": { + "database": "default", + "fields": [ + "host", + "program", + "logdatetime", + "priority", + "message" + ], + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "logdatetime", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "DateTime", + "value": "TODAY" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "program", + "operator": "=", + "type": "String", + "value": "${program:singlequote}" + }, + { + "condition": "AND", + "filterType": "custom", + "key": "message", + "operator": "LIKE", + "type": "String", + "value": "$filter1" + } + ], + "limit": 5000, + "metrics": [], + "mode": "list", + "orderBy": [ + { + "dir": "DESC", + "name": "logdatetime" + } + ], + "table": "log_docker_raw", + "timeField": "logdatetime", + "timeFieldType": "DateTime" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "format": 1, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "builder", + "rawSql": "SELECT \"host\", \"program\", \"logdatetime\", \"priority\", \"message\" FROM \"default\".\"log_docker_raw\" WHERE ( logdatetime >= $__fromTime AND logdatetime <= $__toTime ) AND ( program = ${program:singlequote} ) AND ( message LIKE '%$filter1%' ) ORDER BY logdatetime DESC LIMIT 5000", + "refId": "A", + "selectedFormat": 4 + } + ], + "title": "Panel Title", + "type": "table" + } + ], + "title": "Logs", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "telegram-mg_telegram-1", + "value": "telegram-mg_telegram-1" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "f70e773b-26c3-4b74-9a45-291fab27524b" + }, + "definition": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "hide": 0, + "includeAll": false, + "label": "Program", + "multi": false, + "name": "program", + "options": [], + "query": "SELECT DISTINCT program FROM default.log_docker_raw WHERE logdatetime > now() - interval 6 hours;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "DEBUG", + "value": "DEBUG" + }, + "hide": 0, + "includeAll": false, + "label": "Priority", + "multi": false, + "name": "priority", + "options": [ + { + "selected": false, + "text": ".*", + "value": ".*" + }, + { + "selected": true, + "text": "DEBUG", + "value": "DEBUG" + }, + { + "selected": false, + "text": "INFO", + "value": "INFO" + }, + { + "selected": false, + "text": "WARN", + "value": "WARN" + }, + { + "selected": false, + "text": "ERROR", + "value": "ERROR" + } + ], + "query": ".*,DEBUG,INFO,WARN,ERROR", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "Filter #1", + "name": "filter1", + "options": [ + { + "selected": false, + "text": "*", + "value": "*" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Logs + Structured via transforms", + "uid": "ea7584d7-a2bd-406f-a15e-59b4906bca55", + "version": 19, + "weekStart": "" +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5131ab6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3.8' + +services: + grafana: + image: grafana/grafana:latest + volumes: + - grafana_data:/var/lib/grafana + ports: + - '3000:3000' + clickhouse: + image: docker.io/bitnami/clickhouse:23 + environment: + - CLICKHOUSE_ADMIN_USER=default + - CLICKHOUSE_ADMIN_PASSWORD=default + ports: + - '8123:8123' + volumes: + - clickhouse_data:/bitnami/clickhouse + vector: + image: timberio/vector:latest-alpine + ports: + - '8686:8686' + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./files/vector.toml:/etc/vector/vector.toml:ro + +volumes: + clickhouse_data: + driver: local + grafana_data: + driver: local \ No newline at end of file diff --git a/files/migrations/1698135711_create_logs_table.down.sql b/files/migrations/1698135711_create_logs_table.down.sql new file mode 100644 index 0000000..b1c111a --- /dev/null +++ b/files/migrations/1698135711_create_logs_table.down.sql @@ -0,0 +1 @@ +DROP TABLE log_docker_raw; \ No newline at end of file diff --git a/files/migrations/1698135711_create_logs_table.up.sql b/files/migrations/1698135711_create_logs_table.up.sql new file mode 100644 index 0000000..dc31553 --- /dev/null +++ b/files/migrations/1698135711_create_logs_table.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE log_docker_raw ( + host String, + logdatetime DateTime PRIMARY KEY, + message String, + priority String, + program String +) Engine = MergeTree(); \ No newline at end of file diff --git a/files/vector.toml b/files/vector.toml new file mode 100644 index 0000000..a53054a --- /dev/null +++ b/files/vector.toml @@ -0,0 +1,38 @@ +[api] +enabled = true +address = "0.0.0.0:8686" + +[sources.docker] +type = "docker_logs" +exclude_containers = [ "clickhouse", "grafana", "vector" ] + +[transforms.ts_and_program] +type = "remap" +inputs = [ "docker" ] +source = ''' + .logdatetime = to_unix_timestamp(now()) + .program = .container_name +''' + +[transforms.emptyprogram] +type = "filter" +inputs = [ "ts_and_program" ] +condition.type = "vrl" +condition.source = ''' + .program != "" +''' + +[transforms.dedupe] +type = "dedupe" +inputs = [ "emptyprogram" ] + +[sinks.clickhouse] +type = "clickhouse" +inputs = [ "dedupe"] +compression = "gzip" +endpoint = "http://clickhouse:8123" +auth.strategy = "basic" +auth.user = "default" +auth.password = "default" +database = "default" +table = "log_docker_raw" \ No newline at end of file