diff --git a/traefik_mesh/assets/logs/traefik.yaml b/traefik_mesh/assets/logs/traefik.yaml new file mode 100644 index 0000000000000..a78f97a426b7c --- /dev/null +++ b/traefik_mesh/assets/logs/traefik.yaml @@ -0,0 +1,232 @@ +id: traefik +metric_id: traefik-mesh +backend_only: false +installation_sources: + - traefik + - traefik_mesh + - mesh +facets: + - facetType: range + groups: + - Measure + name: Duration + path: duration + source: log + type: double + unit: + family: time + name: nanosecond + - groups: + - Web Access + name: Method + path: http.method + source: log + - groups: + - Web Access + name: Referer + path: http.referer + source: log + - groups: + - Web Access + name: Status Code + path: http.status_code + source: log + - groups: + - Web Access + name: URL Host + path: http.url_details.host + source: log + - groups: + - Web Access + name: URL Path + path: http.url_details.path + source: log + - groups: + - Web Access + name: URL Port + path: http.url_details.port + source: log + - groups: + - Web Access + name: URL scheme + path: http.url_details.scheme + source: log + - groups: + - Web Access + name: Browser + path: http.useragent_details.browser.family + source: log + - groups: + - Web Access + name: Device + path: http.useragent_details.device.family + source: log + - groups: + - Web Access + name: OS + path: http.useragent_details.os.family + source: log + - groups: + - Web Access + name: Client IP + path: network.client.ip + source: log + - groups: + - Web Access + name: Client Port + path: network.client.port + source: log +pipeline: + type: pipeline + name: Traefik Mesh + enabled: true + filter: + query: source:(traefik OR traefik_mesh OR mesh) + processors: + - type: grok-parser + name: Parsing Traefik logs + enabled: true + source: message + samples: + - 10.32.0.1 - - [07/Dec/2018:06:07:03 +0000] "GET / HTTP/1.1" 302 5 "-" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" 132170 "entrypoint redirect for http" "/" 0ms + - time="2019-04-05T11:57:47Z" level=info msg="Skipping same configuration for provider consul_catalog" + grok: + supportRules: | + _duration %{number:duration:scale(1000000)} + _traefik_backend_url %{regex("[^\\\"“]*"):traefik.backend_url} + _traefik_name %{regex("[^\\\"]*"):traefik.name} + _total_request %{number:traefik.request_total} + _auth %{notSpace:http.auth:nullIf("-")} + _bytes_written %{integer:network.bytes_written} + _client_ip %{ipOrHost:network.client.ip} + _version HTTP\/%{regex("\\d+\\.\\d+"):http.version} + _url %{notSpace:http.url} + _ident %{notSpace:http.ident:nullIf("-")} + _user_agent %{regex("[^\\\"]*"):http.useragent} + _referer %{notSpace:http.referer} + _status_code %{integer:http.status_code} + _method %{word:http.method} + _date_access %{date("dd/MMM/yyyy:HH:mm:ss Z"):date_access} + matchRules: | + access.common %{_client_ip} %{_ident} %{_auth} \[%{_date_access}\] "(?>%{_method} |)%{_url}(?> %{_version}|)" %{_status_code} (?>%{_bytes_written}|-) "%{_referer}" "%{_user_agent}" %{_total_request} "?%{_traefik_name}"? "?%{_traefik_backend_url}["“]? %{_duration}ms.* + + keyvalue %{data::keyvalue} + + - type: message-remapper + name: Define `msg` as the official message of the log + enabled: true + sources: + - msg + - type: attribute-remapper + name: Map `RequestMethod` to `http.method` + enabled: true + sources: + - RequestMethod + target: http.method + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `Duration` to `duration` + enabled: true + sources: + - Duration + target: duration + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `ClientHost` to `network.client.ip` + enabled: true + sources: + - ClientHost + target: network.client.ip + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `ClientPort` to `network.client.port` + enabled: true + sources: + - ClientPort + target: network.client.port + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `OriginStatus`, `DownstreamStatus` to `http.status_code` + enabled: true + sources: + - OriginStatus + - DownstreamStatus + target: http.status_code + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `RequestContentSize` to `network.bytes_read` + enabled: true + sources: + - RequestContentSize + target: network.bytes_read + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: attribute-remapper + name: Map `DownstreamContentSize` to `network.bytes_written` + enabled: true + sources: + - DownstreamContentSize + target: network.bytes_written + preserveSource: false + overrideOnConflict: false + sourceType: attribute + targetType: attribute + - type: url-parser + name: '' + enabled: true + sources: + - http.url + - RequestPath + target: http.url_details + - type: user-agent-parser + name: '' + enabled: true + sources: + - http.useragent + target: http.useragent_details + encoded: false + - type: date-remapper + name: Define `date_access` as the official date of the log + enabled: true + sources: + - date_access + - type: category-processor + name: Categorise status code + enabled: true + categories: + - filter: + query: '@http.status_code:[200 TO 299]' + name: OK + - filter: + query: '@http.status_code:[300 TO 399]' + name: notice + - filter: + query: '@http.status_code:[400 TO 499]' + name: warning + - filter: + query: '@http.status_code:[500 TO 599]' + name: error + target: http.status_category + - type: status-remapper + name: Define `http.status_category`, `level` as the official status of the log + enabled: true + sources: + - http.status_category + - level \ No newline at end of file diff --git a/traefik_mesh/assets/logs/traefik_tests.yaml b/traefik_mesh/assets/logs/traefik_tests.yaml new file mode 100644 index 0000000000000..9e4e30e944404 --- /dev/null +++ b/traefik_mesh/assets/logs/traefik_tests.yaml @@ -0,0 +1,244 @@ +id: "traefik" +tests: + - + sample: "10.32.0.1 - - [07/Dec/2018:06:07:03 +0000] \"GET / HTTP/1.1\" 302 5 \"-\" \"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 132170 \"entrypoint redirect for http\" \"/“ 0ms" + result: + custom: + date_access: 1544162823000 + duration: 0.0 + http: + method: "GET" + referer: "-" + status_category: "notice" + status_code: 302 + url: "/" + url_details: + path: "/" + useragent: "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" + useragent_details: + browser: + family: "Chrome" + major: "51" + minor: "0" + patch: "2704" + patch_minor: "103" + device: + category: "Desktop" + family: "Other" + os: + family: "Windows" + major: "10" + version: "1.1" + network: + bytes_written: 5 + client: + ip: "10.32.0.1" + traefik: + backend_url: "/" + name: "entrypoint redirect for http" + request_total: 132170.0 + message: "10.32.0.1 - - [07/Dec/2018:06:07:03 +0000] \"GET / HTTP/1.1\" 302 5 \"-\" \"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\" 132170 \"entrypoint redirect for http\" \"/“ 0ms" + status: "notice" + tags: + - "source:LOGS_SOURCE" + timestamp: 1544162823000 + - + sample: "10.32.0.1 - - [07/Dec/2018:04:36:29 +0000] \"GET / HTTP/1.1\" 302 5 \"-\" \"HTTP Banner Detection (https://security.ipip.net)\" 132167 \"entrypoint redirect for http\" \"/“ 0ms" + result: + custom: + date_access: 1544157389000 + duration: 0.0 + http: + method: "GET" + referer: "-" + status_category: "notice" + status_code: 302 + url: "/" + url_details: + path: "/" + useragent: "HTTP Banner Detection (https://security.ipip.net)" + useragent_details: + browser: + family: "Other" + device: + brand: "Generic" + category: "Other" + family: "Generic Feature Phone" + model: "Feature Phone" + os: + family: "Other" + version: "1.1" + network: + bytes_written: 5 + client: + ip: "10.32.0.1" + traefik: + backend_url: "/" + name: "entrypoint redirect for http" + request_total: 132167.0 + message: "10.32.0.1 - - [07/Dec/2018:04:36:29 +0000] \"GET / HTTP/1.1\" 302 5 \"-\" \"HTTP Banner Detection (https://security.ipip.net)\" 132167 \"entrypoint redirect for http\" \"/“ 0ms" + status: "notice" + tags: + - "source:LOGS_SOURCE" + timestamp: 1544157389000 + - + sample: "time=\"2019-04-05T11:57:47Z\" level=info msg=\"Skipping same configuration for provider consul_catalog\"" + result: + custom: + level: "info" + time: "2019-04-05T11:57:47Z" + message: "Skipping same configuration for provider consul_catalog" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1 + - + sample: |- + { + "RequestPath" : "https://mail.google.com/mail/u/0/#inbox", + "ClientAddr" : "180.150.154.44:5051", + "RequestHost" : "https://datadoghq.com", + "RequestMethod" : "GET", + "ClientPort" : "5051", + "RequestProtocol" : "1.1", + "BackendAddr" : "127.0.0.1:8088", + "OriginStatusLine" : "404 page not found", + "RequestCount" : 2, + "BackendName" : "unicorn-backend", + "OriginContentSize" : 257, + "OriginStatus" : 404, + "RequestContentSize" : 456, + "BackendURL" : { + "ForceQuery" : false, + "Host" : "127.0.0.1:8088" + }, + "RetryAttempts" : 2, + "ClientUsername" : "-", + "DownstreamStatus" : 404, + "RequestPort" : "6014", + "Duration" : 12345678, + "OriginDuration" : 1345642, + "RequestAddr" : "127.150.32.44:10514", + "ClientHost" : "180.150.154.44", + "RequestLine" : "GET https://mail.google.com/mail/u/0/#inbox HTTP/1.1", + "DownstreamStatusLine" : "404 page not found", + "DownstreamContentSize" : 203, + "Overhead" : 100, + "GzipRatio" : 5, + "FrontendName" : "this-is-my-frontend-name" + } + result: + custom: + BackendAddr: "127.0.0.1:8088" + BackendName: "unicorn-backend" + BackendURL: + ForceQuery: false + Host: "127.0.0.1:8088" + ClientAddr: "180.150.154.44:5051" + ClientUsername: "-" + DownstreamStatusLine: "404 page not found" + FrontendName: "this-is-my-frontend-name" + GzipRatio: 5 + OriginContentSize: 257 + OriginDuration: 1345642 + OriginStatusLine: "404 page not found" + Overhead: 100 + RequestAddr: "127.150.32.44:10514" + RequestCount: 2 + RequestHost: "https://datadoghq.com" + RequestLine: "GET https://mail.google.com/mail/u/0/#inbox HTTP/1.1" + RequestPath: "https://mail.google.com/mail/u/0/#inbox" + RequestPort: "6014" + RequestProtocol: "1.1" + RetryAttempts: 2 + duration: 12345678 + http: + method: "GET" + status_category: "warning" + status_code: 404 + url_details: + hash: "inbox" + host: "mail.google.com" + path: "/mail/u/0/" + scheme: "https" + network: + bytes_read: 456 + bytes_written: 203 + client: + ip: "180.150.154.44" + port: "5051" + message: |- + { + "RequestPath" : "https://mail.google.com/mail/u/0/#inbox", + "ClientAddr" : "180.150.154.44:5051", + "RequestHost" : "https://datadoghq.com", + "RequestMethod" : "GET", + "ClientPort" : "5051", + "RequestProtocol" : "1.1", + "BackendAddr" : "127.0.0.1:8088", + "OriginStatusLine" : "404 page not found", + "RequestCount" : 2, + "BackendName" : "unicorn-backend", + "OriginContentSize" : 257, + "OriginStatus" : 404, + "RequestContentSize" : 456, + "BackendURL" : { + "ForceQuery" : false, + "Host" : "127.0.0.1:8088" + }, + "RetryAttempts" : 2, + "ClientUsername" : "-", + "DownstreamStatus" : 404, + "RequestPort" : "6014", + "Duration" : 12345678, + "OriginDuration" : 1345642, + "RequestAddr" : "127.150.32.44:10514", + "ClientHost" : "180.150.154.44", + "RequestLine" : "GET https://mail.google.com/mail/u/0/#inbox HTTP/1.1", + "DownstreamStatusLine" : "404 page not found", + "DownstreamContentSize" : 203, + "Overhead" : 100, + "GzipRatio" : 5, + "FrontendName" : "this-is-my-frontend-name" + } + status: "warn" + tags: + - "source:LOGS_SOURCE" + timestamp: 1 + - + sample: "127.0.0.1 - - [07/Dec/2018:04:36:29 +0000] \"GET /ping HTTP/1.1\" 200 2 \"-\" \"-\" 1 - - 0ms" + result: + custom: + date_access: 1544157389000 + duration: 0.0 + http: + method: "GET" + referer: "-" + status_category: "OK" + status_code: 200 + url: "/ping" + url_details: + path: "/ping" + useragent: "-" + useragent_details: + browser: + family: "Other" + device: + category: "Other" + family: "Other" + os: + family: "Other" + version: "1.1" + network: + bytes_written: 2 + client: + ip: "127.0.0.1" + traefik: + backend_url: "-" + name: "-" + request_total: 1.0 + message: "127.0.0.1 - - [07/Dec/2018:04:36:29 +0000] \"GET /ping HTTP/1.1\" 200 2 \"-\" \"-\" 1 - - 0ms" + status: "ok" + tags: + - "source:LOGS_SOURCE" + timestamp: 1544157389000 \ No newline at end of file diff --git a/traefik_mesh/assets/saved_views/traefik_mesh_error_overview.json b/traefik_mesh/assets/saved_views/traefik_mesh_error_overview.json new file mode 100644 index 0000000000000..1c21d2d835cc1 --- /dev/null +++ b/traefik_mesh/assets/saved_views/traefik_mesh_error_overview.json @@ -0,0 +1,14 @@ +{ + "name": "Traefik Mesh Error Logs Overview", + "query": "source:traefik_mesh status:error", + "visible_facets": [ + "source", + "host", + "service" + ], + "type": "logs", + "timerange": { + "interval_ms": 3600000 + }, + "page": "patterns" +} \ No newline at end of file diff --git a/traefik_mesh/assets/saved_views/traefik_mesh_log_overview.json b/traefik_mesh/assets/saved_views/traefik_mesh_log_overview.json new file mode 100644 index 0000000000000..b0b02a61a0989 --- /dev/null +++ b/traefik_mesh/assets/saved_views/traefik_mesh_log_overview.json @@ -0,0 +1,14 @@ +{ + "name": "Traefik Mesh Logs Overview", + "query": "source:traefik_mesh", + "visible_facets": [ + "source", + "host", + "service" + ], + "type": "logs", + "timerange": { + "interval_ms": 3600000 + }, + "page": "patterns" +} \ No newline at end of file diff --git a/traefik_mesh/manifest.json b/traefik_mesh/manifest.json index 5a56ac0e0599c..06dba0cf29eca 100644 --- a/traefik_mesh/manifest.json +++ b/traefik_mesh/manifest.json @@ -48,8 +48,9 @@ "monitors": { "High Request Count": "assets/monitors/high_request_count.json" }, - "logs": { - "source": "traefik_mesh" + "saved_views": { + "Traefik Mesh Logs Overview": "assets/saved_views/traefik_mesh_log_overview.json", + "Traefik Mesh Error Logs Overview": "assets/saved_views/traefik_mesh_error_overview.json" } }, "author": {