From faae826f6f2d7dec5e0d96356211b3574f139114 Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 12:54:47 -0700 Subject: [PATCH 1/2] Added support of field list, output format (text or json), custom delay between Graylog API calls. --- .gitignore | 1 + gtail/gtail.py | 162 ++++++++++++++++++++++++++++++------------------- 2 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/gtail/gtail.py b/gtail/gtail.py index b6951ce..063430a 100755 --- a/gtail/gtail.py +++ b/gtail/gtail.py @@ -12,6 +12,7 @@ import sys import time import urllib +from json import dumps MAX_DELAY = 10 DEFAULT_CONFIG_PATHS = [".gtail", os.path.expanduser("~/.gtail")] @@ -60,10 +61,13 @@ def list_streams(streams): def fetch_messages(server_config, query = None, stream_ids = None, - last_message_id = None): + last_message_id = None, + fields = None, + delay = MAX_DELAY): url = [] url.append(server_config.uri) - url.append("/search/universal/relative?range=7200&limit=100") + range = max(delay*5, 300) + url.append("/search/universal/relative?range={range}&limit=1000".format(range=range)) # query terms if query: @@ -71,6 +75,12 @@ def fetch_messages(server_config, else: url.append("&query=*") + # fields list + if fields: + if "_id" not in fields: + fields.append("_id") + url.append("&fields=" + "%2C".join(fields)) + # stream ID if stream_ids: quoted = map(urllib.quote_plus, stream_ids) @@ -109,37 +119,46 @@ def fetch_messages(server_config, # pretty prints a message # streams, if provided, is the full list of streams; it is used for pretty # printing of the stream name -def print_message(message, streams=None): - s = [] - if "timestamp" in message: - timestamp = message["timestamp"] - s.append(timestamp) - if streams and "streams" in message: - stream_ids = message["streams"] - stream_names = [] - for sid in stream_ids: - stream_names.append(streams[sid]["title"]) - s.append("[" + ", ".join(stream_names) + "]") - if "facility" in message: - facility = message["facility"] - s.append(facility) - if "level" in message: - level = message["level"] - s.append(level) - if "source" in message: - source = message["source"] - s.append(source) - if "loggerName" in message: - logger_name = message["loggerName"] - s.append(logger_name) - - if "full_message" in message: - text = message["full_message"] +def print_message(message, streams=None, fields=None, format="json"): + s = dict() + text = None + if fields: + count = 0 + for field in fields: + if field != "_id" and field in message: + count += 1 + s[field] = message[field] + else: + if "timestamp" in message: + s["timestamp"] = message["timestamp"] + if streams and "streams" in message: + stream_ids = message["streams"] + stream_names = [] + for sid in stream_ids: + stream_names.append(streams[sid]["title"]) + s["streams"] = "[" + ", ".join(stream_names) + "]" + if "facility" in message: + s["facility"] = message["facility"] + if "level" in message: + s["level"] = message["level"] + if "source" in message: + s["source"] = message["source"] + if "loggerName" in message: + s["loggerName"] = message["loggerName"] + + if "full_message" in message: + text = message["full_message"] + elif "message" in message: + text = message["message"] + + if format == "text": + out = map(str, s.values()) else: - text = message["message"] + out = dumps(s) + print bold(out) - print bold(" ".join(map(str, s))) - print text + if text: + print text # config object and config parsing Config = namedtuple("Config", "server_config") @@ -234,6 +253,15 @@ def main(): parser.add_argument("--query", dest="query", nargs="+", help="Query terms to search on") + parser.add_argument("--fields", dest="fields", + nargs="+", + help="Fields to display") + parser.add_argument("--format", dest="format", + choices=["text", "json"], default="json", + help="Display format") + parser.add_argument("--delay", dest="delay", + type=int, default=MAX_DELAY, + help="Delay between Rest API calls (seconds)") parser.add_argument("--config", dest="config_paths", nargs="+", help="Config files. Default: " + ", ".join(DEFAULT_CONFIG_PATHS)) @@ -280,37 +308,47 @@ def main(): # print log messages # - last_message_id = None - while True: - # time-forward messages - query = None - if args.query: - query = ' '.join(args.query) - try: - messages = fetch_messages( - server_config = server_config, - query = query, - stream_ids = stream_ids, - last_message_id = last_message_id) - except Exception as e: - print e - time.sleep(MAX_DELAY) - continue - - # print new messages - last_timestamp = None - for m in messages: - print_message(m, streams) - last_message_id = m["_id"] - last_timestamp = m["timestamp"] - - if last_timestamp: - seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds()) - delay = min(seconds_since_last_message, MAX_DELAY) - if delay > 2: - time.sleep(delay) - else: - time.sleep(MAX_DELAY) + try: + last_message_id = None + while True: + # time-forward messages + query = None + fields = None + if args.query: + query = ' '.join(args.query) + if args.fields: + fields = [] + for field in args.fields: + fields.extend(field.split(",")) + try: + messages = fetch_messages( + server_config = server_config, + query = query, + stream_ids = stream_ids, + last_message_id = last_message_id, + fields=fields, + delay=args.delay) + except Exception as e: + print e + time.sleep(args.delay) + continue + + # print new messages + last_timestamp = None + for m in messages: + print_message(m, streams, fields=fields, format=args.format) + last_message_id = m["_id"] + last_timestamp = m["timestamp"] + + if last_timestamp: + seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds()) + delay = min(seconds_since_last_message, args.delay) + if delay > 2: + time.sleep(delay) + else: + time.sleep(args.delay) + except KeyboardInterrupt: + os._exit(0) if __name__ == "__main__": rc = main() From 88ec3a861acc25f4274f876488871e1b0f02030f Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 12:58:37 -0700 Subject: [PATCH 2/2] Updated documentation --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6845e1a..a54c166 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,18 @@ This uses Graylog's search mechanism, so you can use `field: value` syntax. `gtail --stream nile --query crocodile` +### Select fields + +`gtail --fields comma-separated-fields-here` + +This will display only selected fields. + +### Display format + +`gtail --format [text or json]` + +Choose between comma separated values map and JSON output. + ### Full usage instructions `gtail --help`