From e1173755cd1c9293addd7ddf348d7553a31daf35 Mon Sep 17 00:00:00 2001 From: Sergey Belyaev Date: Thu, 10 Sep 2020 23:50:31 +0700 Subject: [PATCH] Implement LOG_FORMAT=json Centralized log collection system works better with logs in json format. E.g. DataDog strongly encourage setting up your logging library to produce your logs in JSON format to avoid the need for custom parsing rules. So, the next small fix is all we need to get json logs. Signed-off-by: Sergey Belyaev --- README.md | 35 ++++++++++++++++++++++++++++++++ src/service_cmd/runner/runner.go | 11 ++++++++++ src/settings/settings.go | 1 + 3 files changed, 47 insertions(+) diff --git a/README.md b/README.md index 304e835d2..ff38814ee 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ - [Example 3](#example-3) - [Example 4](#example-4) - [Loading Configuration](#loading-configuration) + - [Log Format](#log-format) - [Request Fields](#request-fields) - [Statistics](#statistics) - [HTTP Port](#http-port) @@ -347,6 +348,40 @@ The former is the default behavior. To use the latter method, set the `RUNTIME_W For more information on how runtime works you can read its [README](https://github.com/lyft/goruntime). +## Log Format + +A centralized log collection system works better with logs in json format. JSON format avoids the need for custom parsing rules. +The Ratelimit service produces logs in a text format by default. For Example: + +``` +time="2020-09-10T17:22:35Z" level=debug msg="loading domain: messaging" +time="2020-09-10T17:22:35Z" level=debug msg="loading descriptor: key=messaging.message_type_marketing" +time="2020-09-10T17:22:35Z" level=debug msg="loading descriptor: key=messaging.message_type_marketing.to_number ratelimit={requests_per_unit=5, unit=DAY}" +time="2020-09-10T17:22:35Z" level=debug msg="loading descriptor: key=messaging.to_number ratelimit={requests_per_unit=100, unit=DAY}" +time="2020-09-10T17:21:55Z" level=warning msg="Listening for debug on ':6070'" +time="2020-09-10T17:21:55Z" level=warning msg="Listening for HTTP on ':8080'" +time="2020-09-10T17:21:55Z" level=debug msg="waiting for runtime update" +time="2020-09-10T17:21:55Z" level=warning msg="Listening for gRPC on ':8081'" +``` + +JSON Log format can be configured using the following environment variables: + +``` +LOG_FORMAT=json +``` + +Output example: +``` +{"@message":"loading domain: messaging","@timestamp":"2020-09-10T17:22:44.926010192Z","level":"debug"} +{"@message":"loading descriptor: key=messaging.message_type_marketing","@timestamp":"2020-09-10T17:22:44.926019315Z","level":"debug"} +{"@message":"loading descriptor: key=messaging.message_type_marketing.to_number ratelimit={requests_per_unit=5, unit=DAY}","@timestamp":"2020-09-10T17:22:44.926037174Z","level":"debug"} +{"@message":"loading descriptor: key=messaging.to_number ratelimit={requests_per_unit=100, unit=DAY}","@timestamp":"2020-09-10T17:22:44.926048993Z","level":"debug"} +{"@message":"Listening for debug on ':6070'","@timestamp":"2020-09-10T17:22:44.926113905Z","level":"warning"} +{"@message":"Listening for gRPC on ':8081'","@timestamp":"2020-09-10T17:22:44.926182006Z","level":"warning"} +{"@message":"Listening for HTTP on ':8080'","@timestamp":"2020-09-10T17:22:44.926227031Z","level":"warning"} +{"@message":"waiting for runtime update","@timestamp":"2020-09-10T17:22:44.926267808Z","level":"debug"} +``` + # Request Fields For information on the fields of a Ratelimit gRPC request please read the information diff --git a/src/service_cmd/runner/runner.go b/src/service_cmd/runner/runner.go index ef318bde1..80e8e7814 100644 --- a/src/service_cmd/runner/runner.go +++ b/src/service_cmd/runner/runner.go @@ -4,6 +4,7 @@ import ( "io" "math/rand" "net/http" + "strings" "time" stats "github.com/lyft/gostats" @@ -43,6 +44,16 @@ func (runner *Runner) Run() { } else { logger.SetLevel(logLevel) } + if strings.ToLower(s.LogFormat) == "json" { + logger.SetFormatter(&logger.JSONFormatter{ + TimestampFormat: time.RFC3339Nano, + FieldMap: logger.FieldMap{ + logger.FieldKeyTime: "@timestamp", + logger.FieldKeyMsg: "@message", + }, + }) + } + var localCache *freecache.Cache if s.LocalCacheSizeInBytes != 0 { localCache = freecache.NewCache(s.LocalCacheSizeInBytes) diff --git a/src/settings/settings.go b/src/settings/settings.go index 78e902426..3e951b389 100644 --- a/src/settings/settings.go +++ b/src/settings/settings.go @@ -22,6 +22,7 @@ type Settings struct { RuntimeIgnoreDotFiles bool `envconfig:"RUNTIME_IGNOREDOTFILES" default:"false"` RuntimeWatchRoot bool `envconfig:"RUNTIME_WATCH_ROOT" default:"true"` LogLevel string `envconfig:"LOG_LEVEL" default:"WARN"` + LogFormat string `envconfig:"LOG_FORMAT" default:"text"` RedisSocketType string `envconfig:"REDIS_SOCKET_TYPE" default:"unix"` RedisUrl string `envconfig:"REDIS_URL" default:"/var/run/nutcracker/ratelimit.sock"` RedisPoolSize int `envconfig:"REDIS_POOL_SIZE" default:"10"`