diff --git a/CHANGELOG.md b/CHANGELOG.md index fa81bc42b7..30b930c55a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added support for parent.config markdown/retry DS parameters using first./inner./last. prefixes. mso. and prefixes should be deprecated. - Add new __REGEX_REMAP_DIRECTIVE__ support to raw remap text to allow moving the regex_remap placement. - t3c change `t3c diff` call to `t3c-diff` to fix a performance regression. +- Added a sub-app t3c-tail to tail diags.log and capture output when t3c reloads and restarts trafficserver ### Fixed - Fixed TO to default route ID to 0, if it is not present in the request context. diff --git a/cache-config/Makefile b/cache-config/Makefile index bc871d0ce4..475190f822 100644 --- a/cache-config/Makefile +++ b/cache-config/Makefile @@ -24,7 +24,7 @@ TC_VERSION ?= $(shell cat ../VERSION) GO_FLAGS ?= PANDOC_FLAGS := --strip-comments -TARGETS := t3c/t3c t3c-apply/t3c-apply t3c-check/t3c-check t3c-check-refs/t3c-check-refs t3c-check-reload/t3c-check-reload t3c-diff/t3c-diff t3c-generate/t3c-generate t3c-preprocess/t3c-preprocess t3c-request/t3c-request t3c-update/t3c-update +TARGETS := t3c/t3c t3c-apply/t3c-apply t3c-check/t3c-check t3c-check-refs/t3c-check-refs t3c-check-reload/t3c-check-reload t3c-diff/t3c-diff t3c-generate/t3c-generate t3c-preprocess/t3c-preprocess t3c-request/t3c-request t3c-tail/t3c-tail t3c-update/t3c-update .PHONY: debug all man rst clean @@ -81,6 +81,8 @@ t3c-preprocess/t3c-preprocess: $(wildcard t3c-preprocess/**/*.go) $(wildcard t3c go build -o $@ $(GO_FLAGS) github.com/apache/trafficcontrol/cache-config/$(dir $@) t3c-request/t3c-request: $(wildcard t3c-request/**/*.go) $(wildcard t3c-request/*.go) go build -o $@ $(GO_FLAGS) github.com/apache/trafficcontrol/cache-config/$(dir $@) +t3c-tail/t3c-tail: $(wildcard t3c-tail/**/*.go) $(wildcard t3c-tail/*.go) + go build -o $@ $(GO_FLAGS) github.com/apache/trafficcontrol/cache-config/$(dir $@) t3c-update/t3c-update: $(wildcard t3c-update/**/*.go) $(wildcard t3c-update/*.go) go build -o $@ $(GO_FLAGS) github.com/apache/trafficcontrol/cache-config/$(dir $@) diff --git a/cache-config/build/build_rpm.sh b/cache-config/build/build_rpm.sh index c022da9ace..3e80f1ed33 100755 --- a/cache-config/build/build_rpm.sh +++ b/cache-config/build/build_rpm.sh @@ -115,6 +115,12 @@ initBuildArea() { buildManpage 't3c-diff'; ) + ( + cd t3c-tail; + go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" -tags "$tags"; + buildManpage 't3c-tail'; + ) + ( cd t3c-preprocess; go build -v -gcflags "$gcflags" -ldflags "${ldflags} -X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" -tags "$tags"; diff --git a/cache-config/build/trafficcontrol-cache-config.spec b/cache-config/build/trafficcontrol-cache-config.spec index 04b57f901d..607758743b 100644 --- a/cache-config/build/trafficcontrol-cache-config.spec +++ b/cache-config/build/trafficcontrol-cache-config.spec @@ -123,6 +123,14 @@ go_t3c_preprocess_dir="$ccpath"/t3c-preprocess cp "$TC_DIR"/"$ccdir"/t3c-preprocess/t3c-preprocess.1 . ) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } +# copy t3c-tail binary +go_t3c_tail_dir="$ccpath"/t3c-tail +( mkdir -p "$go_t3c_tail_dir" && \ + cd "$go_t3c_tail_dir" && \ + cp "$TC_DIR"/"$ccdir"/t3c-tail/t3c-tail . + cp "$TC_DIR"/"$ccdir"/t3c-tail/t3c-tail.1 . +) || { echo "Could not copy go program at $(pwd): $!"; exit 1; } + %install ccdir="cache-config/" installdir="/usr/bin" @@ -165,6 +173,10 @@ t3c_diff_src=src/github.com/apache/trafficcontrol/"$ccdir"/t3c-diff cp -p "$t3c_diff_src"/t3c-diff ${RPM_BUILD_ROOT}/"$installdir" gzip -c -9 "$src"/t3c-diff/t3c-diff.1 > ${RPM_BUILD_ROOT}/"$mandir"/"$man1dir"/t3c-diff.1.gz +t3c_tail_src=src/github.com/apache/trafficcontrol/"$ccdir"/t3c-tail +cp -p "$t3c_tail_src"/t3c-tail ${RPM_BUILD_ROOT}/"$installdir" +gzip -c -9 "$src"/t3c-tail/t3c-tail.1 > ${RPM_BUILD_ROOT}/"$mandir"/"$man1dir"/t3c-tail.1.gz + t3c_check_src=src/github.com/apache/trafficcontrol/"$ccdir"/t3c-check cp -p "$t3c_check_src"/t3c-check ${RPM_BUILD_ROOT}/"$installdir" gzip -c -9 "$src"/t3c-check/t3c-check.1 > ${RPM_BUILD_ROOT}/"$mandir"/"$man1dir"/t3c-check.1.gz @@ -224,6 +236,7 @@ fi /usr/bin/t3c-generate /usr/bin/t3c-preprocess /usr/bin/t3c-request +/usr/bin/t3c-tail /usr/bin/t3c-update /usr/share/man/man1/t3c.1.gz /usr/share/man/man1/t3c-apply.1.gz @@ -234,6 +247,7 @@ fi /usr/share/man/man1/t3c-generate.1.gz /usr/share/man/man1/t3c-preprocess.1.gz /usr/share/man/man1/t3c-request.1.gz +/usr/share/man/man1/t3c-tail.1.gz /usr/share/man/man1/t3c-update.1.gz %dir /var/lib/trafficcontrol-cache-config diff --git a/cache-config/t3c-apply/torequest/cmd.go b/cache-config/t3c-apply/torequest/cmd.go index d6263292e3..85855d46d0 100644 --- a/cache-config/t3c-apply/torequest/cmd.go +++ b/cache-config/t3c-apply/torequest/cmd.go @@ -33,6 +33,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -59,6 +60,7 @@ type ServerAndConfigs struct { ConfigFiles json.RawMessage } +var stripDate = regexp.MustCompile(`\[\w{3}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{3}\]\s`) var t3cpath string = filepath.Join(t3cutil.InstallDir(), `t3c`) // generate runs t3c-generate and returns the result. @@ -289,6 +291,31 @@ func sendUpdate(cfg config.Cfg, configApplyTime, revalApplyTime *time.Time, conf return nil } +//doTail calls t3c-tail and will run a tail on the log file provided with string for a regex to +//maatch on default is .* endMatch will make t3c-tail exit when a pattern is matched otherwise +//a timeout in a given number of seconds will occur. +func doTail(cfg config.Cfg, file string, logMatch string, endMatch string, timeoutInMS int) error { + args := []string{ + "--file=" + filepath.Join(cfg.TsHome, file), + "--match=" + logMatch, + "--end-match=" + endMatch, + "--timeout-ms=" + strconv.Itoa(timeoutInMS), + } + stdOut, stdErr, code := t3cutil.Do(`t3c-tail`, args...) + if code > 1 { + return fmt.Errorf("t3c-tail returned error code %v stdout '%v' stderr '%v'", code, string(stdOut), string(stdErr)) + } + logSubApp(`t3c-tail`, stdErr) + + stdOut = bytes.TrimSpace(stdOut) + lines := strings.Split(string(stdOut), "\n") + for _, line := range lines { + line = stripDate.ReplaceAllString(line, "") + log.Infoln(line) + } + return nil +} + // diff calls t3c-diff to diff the given new file and the file on disk. Returns whether they're different. // Logs the difference. // If the file on disk doesn't exist, returns true and logs the entire file as a diff. diff --git a/cache-config/t3c-apply/torequest/torequest.go b/cache-config/t3c-apply/torequest/torequest.go index 996506e621..32b6a445a3 100644 --- a/cache-config/t3c-apply/torequest/torequest.go +++ b/cache-config/t3c-apply/torequest/torequest.go @@ -47,6 +47,15 @@ const ( UpdateTropsFailed UpdateStatus = 3 ) +const ( + TailDiagsLogRelative = "/var/log/trafficserver/diags.log" + TailRestartTimeOutMS = 60000 + TailReloadTimeOutMS = 15000 + tailMatch = `ET_(TASK|NET)\s\d{1,}` + tailRestartEnd = "Traffic Server is fully initialized" + tailReloadEnd = "remap.config finished loading" +) + type Package struct { Name string `json:"name"` Version string `json:"version"` @@ -1111,6 +1120,9 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu } t3cutil.WriteActionLog(t3cutil.ActionLogActionATSRestart, t3cutil.ActionLogStatusSuccess, metaData) log.Infoln("trafficserver has been " + startStr + "ed") + if err := doTail(r.Cfg, TailDiagsLogRelative, ".*", tailRestartEnd, TailRestartTimeOutMS); err != nil { + log.Errorln("error running tail") + } if *syncdsUpdate == UpdateTropsNeeded { *syncdsUpdate = UpdateTropsSuccessful } @@ -1137,6 +1149,9 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu *syncdsUpdate = UpdateTropsSuccessful } log.Infoln("ATS 'traffic_ctl config reload' was successful") + if err := doTail(r.Cfg, TailDiagsLogRelative, tailMatch, tailReloadEnd, TailReloadTimeOutMS); err != nil { + log.Errorln("error running tail: ", err) + } } if *syncdsUpdate == UpdateTropsNeeded { *syncdsUpdate = UpdateTropsSuccessful diff --git a/cache-config/t3c-tail/README.md b/cache-config/t3c-tail/README.md new file mode 100644 index 0000000000..0777b67eef --- /dev/null +++ b/cache-config/t3c-tail/README.md @@ -0,0 +1,83 @@ + + + +# NAME + +t3c-tail - Traffic Control Cache Configuration tail tool + +# SYNOPSIS + +t3c-tail \-f \ \-m \ \-e \ \-t \ + +[\-\-help] + +[\-\-version] + +# DESCRIPTION + +The t3c-tail application will tail a file, usually a log file. +Provide a file name to watch, a regex to filter or .* is the default, +a regex match to exit tail (if omitted will exit on timeout), +timeout in milliseconds for how long you want it to run, default is 15000 milliseconds. + +# OPTIONS + +-e, -\-end-match + + Regex pattern that will cause tail to exit before timeout. + +-f, -\-file + Path to file to watch. + +-h, -\-help + + Print usage info and exit. + +-m, -\-match + Regex pattern you want to match while running tail default is .*. + +-t -\-timeout-ms + Timeout in milliseconds that will cause tail to exit default is 15000 MS. + +-V, -\-version + + Print version information and exit. + +# AUTHORS + +The t3c application is maintained by Apache Traffic Control project. For help, bug reports, contributing, or anything else, see: + +https://trafficcontrol.apache.org/ + +https://github.com/apache/trafficcontrol diff --git a/cache-config/t3c-tail/t3c-tail.go b/cache-config/t3c-tail/t3c-tail.go new file mode 100644 index 0000000000..1979de686c --- /dev/null +++ b/cache-config/t3c-tail/t3c-tail.go @@ -0,0 +1,122 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "fmt" + "os" + "regexp" + "time" + + "github.com/apache/trafficcontrol/cache-config/t3cutil" + "github.com/apache/trafficcontrol/lib/go-log" + + "github.com/nxadm/tail" + "github.com/pborman/getopt/v2" +) + +const AppName = "t3c-tail" + +// Version is the application version. +// This is overwritten by the build with the current project version. +var Version = "0.4" + +// GitRevision is the git revision the application was built from. +// This is overwritten by the build with the current project version. +var GitRevision = "nogit" + +// defaultTimeOutMs is 15000 milliseconds, if not included in input. +var defaultTimeOutMs = 15000 + +func main() { + file := getopt.StringLong("file", 'f', "", "Path to file to watch") + match := getopt.StringLong("match", 'm', ".*", "Regex pattern you want to match while running tail default is .*") + endMatch := getopt.StringLong("end-match", 'e', "^timeout", "Regex pattern that will cause tail to exit before timeout") + timeOutMs := getopt.Int64Long("timeout-ms", 't', int64(defaultTimeOutMs), "Timeout in milliseconds that will cause tail to exit default is 15000 MS") + version := getopt.BoolLong("version", 'V', "Print version information and exit.") + help := getopt.BoolLong("help", 'h', "Print usage information and exit") + getopt.Parse() + + log.Init(os.Stderr, os.Stderr, os.Stderr, os.Stderr, os.Stderr) + + if *help { + fmt.Println(usageStr()) + os.Exit(0) + } else if *version { + fmt.Println(t3cutil.VersionStr(AppName, Version, GitRevision)) + os.Exit(0) + } + + if *file == "" || file == nil { + fmt.Println("Please provide file path for t3c-tail") + fmt.Println(usageStr()) + os.Exit(1) + } + + logMatch := regexp.MustCompile(*match) + tailStop := regexp.MustCompile(*endMatch) + timeOut := *timeOutMs + + t, err := tail.TailFile(*file, + tail.Config{ + MustExist: true, + Follow: true, + Location: &tail.SeekInfo{ + Offset: 0, + Whence: 2, + }, + }) + if err != nil { + log.Errorln("error running tail on ", file, err) + os.Exit(1) + } + timer := time.NewTimer(time.Millisecond * time.Duration(timeOut)) + go func() { + for line := range t.Lines { + if logMatch.MatchString(line.Text) { + fmt.Println(line.Text) + } + if tailStop.MatchString(line.Text) { + if !timer.Stop() { + <-timer.C + } + timer.Reset(0) + break + } + } + }() + + <-timer.C + t.Cleanup() +} + +func usageStr() string { + return `usage: t3c-tail [--help] + -f -m -e -t + + file is path to the file you want to tail + + match is regex string you wish to match on, default is '.*' + + endMatch is a regex used to exit tail when it is found in the logs with out waiting for timeout + + timeOutMs is when tail will stop if endMatch isn't found default is 15000 + ` +} diff --git a/cache-config/t3c/t3c.go b/cache-config/t3c/t3c.go index 1488b531bc..774a5bb932 100644 --- a/cache-config/t3c/t3c.go +++ b/cache-config/t3c/t3c.go @@ -48,6 +48,7 @@ var commands = map[string]struct{}{ "generate": struct{}{}, "preprocess": struct{}{}, "request": struct{}{}, + "tail": struct{}{}, "update": struct{}{}, } @@ -116,6 +117,7 @@ These are the available commands: generate generate configuration from Traffic Ops data preprocess preprocess generated config files request request Traffic Ops data + tail tail a log file update update a cache's queue and reval status in Traffic Ops ` }