From dd39135c1fc3089a50f9e6907a18819011fe98d1 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 14:08:10 -0600 Subject: [PATCH 01/49] Added rule to build atstccfg for caches --- infrastructure/cdn-in-a-box/Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index 69c65c60d6..13dcba039a 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -41,8 +41,10 @@ SPECIAL_SAUCE := $(TC_VERSION)-$(BUILD_NUMBER).$(RHEL_VERSION).x86_64.rpm SPECIAL_SEASONING := $(TOMCAT_VERSION).$(TOMCAT_RELEASE)-$(BUILD_NUMBER).$(RHEL_VERSION).x86_64.rpm TO_SOURCE := $(wildcard ../../traffic_ops/**/*) -ORT_SOURCE:= $(wildcard ../../traffic_ops/bin/*) TO_SOURCE += $(wildcard ../../traffic_ops_db/**/*) +ORT_SOURCE:= $(wildcard ../../traffic_ops/ort/atstccfg/**/*.go) +ORT_SOURCE+= $(wildcard ../../traffic_ops/client/*.go) +ORT_SOURCE+= $(wildcard ../../lib/**/*.go) TM_SOURCE := $(wildcard ../../traffic_monitor/**/*) TP_SOURCE := $(wildcard ../../traffic_portal/**/*) TR_SOURCE := $(wildcard ../../traffic_router/**/*) @@ -73,6 +75,8 @@ traffic_router/tomcat.rpm: ../../dist/tomcat-$(SPECIAL_SEASONING) cp -f $? $@ traffic_stats/traffic_stats.rpm: ../../dist/traffic_stats-$(SPECIAL_SAUCE) cp -f $? $@ +cache/atstccfg: $(ORT_SOURCE) + go build -o $@ ../../traffic_ops/ort/atstccfg # Dist rpms ../../dist/traffic_monitor-$(SPECIAL_SAUCE): $(TM_SOURCE) From 7593354f5bc01d8f9dcf9aba9c6688170839ea40 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 14:08:29 -0600 Subject: [PATCH 02/49] Adding atstccfg to cache images --- infrastructure/cdn-in-a-box/cache/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index 125119aa56..d5aae9af0b 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -55,4 +55,5 @@ WORKDIR /opt RUN touch /var/log/ort.log && pip3 install Apache-TrafficControl==1.1.3 && pip3 install ./ort && cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && mkdir -p /opt/init.d && cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / +ADD infrastructure/cdn-in-a-box/cache/atstccfg /bin/ CMD /run.sh From b7c32b01d09486f9ea7d3c867980201fb6c8b9d5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 14:10:29 -0600 Subject: [PATCH 03/49] Added -t/--timeout option (not implemented) --- .../cdn-in-a-box/ort/traffic_ops_ort/__init__.py | 8 ++++++++ .../cdn-in-a-box/ort/traffic_ops_ort/configuration.py | 1 + 2 files changed, 9 insertions(+) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py index 9ebd6280bd..5e8f425fd7 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py @@ -70,6 +70,10 @@ Print version information and exit +.. option:: -t, --timeout + + Sets the timeout in milliseconds for connections to Traffic Ops. + .. option:: -k, --insecure An optional flag which, when used, disables the checking of SSL certificates for validity @@ -484,6 +488,10 @@ def main() -> int: help="Skip verification of SSL certificates for Traffic Ops connections. "\ "DON'T use this in production!", action="store_true") + parser.add_argument("-t", "--timeout", + help="Sets the timeout in milliseconds for requests made to Traffic Ops.", + type=int, + default=None) parser.add_argument("-v", "--version", action="version", version="%(prog)s v"+__version__, diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py index 1d2c1c75bb..84ff9193c8 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py @@ -112,6 +112,7 @@ def __init__(self, args:argparse.Namespace): self.retries = args.retries if args.retries > 0 else 0 self.rev_proxy_disable = args.rev_proxy_disable self.verify = not args.insecure + self.timeout = args.timeout setLogLevel(args.log_level) From c0c755c85b35a094829092b6c743954fad37f4ab Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 14:11:33 -0600 Subject: [PATCH 04/49] Added cache/atstccfg to gitignore --- infrastructure/cdn-in-a-box/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/cdn-in-a-box/.gitignore b/infrastructure/cdn-in-a-box/.gitignore index aea22e038c..774763d981 100644 --- a/infrastructure/cdn-in-a-box/.gitignore +++ b/infrastructure/cdn-in-a-box/.gitignore @@ -18,3 +18,4 @@ *.rpm traffic_ops/ca optional/vpn/vpnca +cache/atstccfg From 728daea349565697642d165b5d6db82cdf474923 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 14:11:55 -0600 Subject: [PATCH 05/49] Switched from API to atstccfg for updating status --- .../ort/traffic_ops_ort/to_api.py | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 175292fcf3..57d4bcfb2d 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -20,9 +20,10 @@ It extends the class provided by the official Apache Traffic Control Client. """ -import typing import logging import re +import subprocess +import typing from munch import Munch from requests.compat import urljoin @@ -71,6 +72,23 @@ def __init__(self, conf:Configuration): self.hostname = conf.shortHostname + self.atstccfgCmd = [ + "atstccfg", + "--traffic-ops-url=http{}://{}:{}".format("s" if conf.useSSL else "", conf.toHost, conf.toPort), + "--cache-host-name={}".format(conf.hostname), + "--traffic-ops-user={}".format(conf.username), + "--traffic-ops-password={}".format(conf.password), + "--log-location-error=stderr", + "--log-location-warning=stderr", + "--log-location-info=stderr" + ] + + if conf.timeout is not None and conf.timeout >= 0: + self.atstccfgCmd.append("--traffic-ops-timeout-milliseconds={}".format(conf.timeout)) + if conf.rev_proxy_disable: + self.atstccfgCmd.append("--traffic-ops-disable-proxy") + + def __enter__(self): """ Implements context-management for :class:`API` objects. Needs to override :class:`TOSession` @@ -199,23 +217,19 @@ def updateTrafficOps(self, mode:Configuration.Modes): if mode is Configuration.Modes.REPORT: return - payload = {"updated": False, "reval_updated": False} - + atstccfgCmd = self.atstccfgCmd + ["--set-queue-status=false", "--set-reval-status=false"] for _ in range(self.retries): try: - response = self._session.post('/'.join((self._server_url.rstrip('/'), - "update", - self.hostname) - ), data=payload) - break + proc = subprocess.run(atstccfgCmd, capture_output=True, text=True) + logging.info(proc.stdout) + logging.error(proc.stderr) + if proc.returncode == 0: + break except (LoginError, InvalidJSONError, OperationError, RequestException) as e: - logging.debug("TO update failure: %r", e, exc_info=True, stack_info=True) + logging.error("TO update failure: %r", e, exc_info=True, stack_info=True) else: raise ConnectionError("Failed to update Traffic Ops - connection was lost") - if response.text: - logging.info("Traffic Ops response: %s", response.text) - def getMyChkconfig(self) -> typing.List[dict]: """ Fetches the 'chkconfig' for this server From 33f6291c4156e1d51d2c555b08e1266039158728 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Mon, 27 Apr 2020 21:09:06 -0600 Subject: [PATCH 06/49] Switched current update status fetch to use atstccfg --- .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 57d4bcfb2d..25c8833a92 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -272,11 +272,11 @@ def getMyUpdateStatus(self) -> dict: logging.info("Fetching update status from Traffic Ops") for _ in range(self.retries): try: - # The API function decorator confuses pylint into thinking this doesn't return - #pylint: disable=E1111 - r = self.get_server_update_status(server_name=self.hostname) - #pylint: enable=E1111 - break + proc = subprocess.run(self.atstccfgCmd + ["--get-data=update-status"]) + logging.info(proc.stdout) + logging.error(proc.stderr) + if proc.returncode == 0: + break; except (InvalidJSONError, LoginError, OperationError, RequestException) as e: logging.debug("update status fetch failure: %r", e, exc_info=True, stack_info=True) else: From 4cf29768c834cc686063ea7478aabc47c096fb15 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 12:09:35 -0600 Subject: [PATCH 07/49] Added implementation of --insecure for atstccfg calls, switched chkconfig fetch to atstccfg --- .../ort/traffic_ops_ort/to_api.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 25c8833a92..53bf463e04 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -20,6 +20,7 @@ It extends the class provided by the official Apache Traffic Control Client. """ +import json import logging import re import subprocess @@ -87,6 +88,8 @@ def __init__(self, conf:Configuration): self.atstccfgCmd.append("--traffic-ops-timeout-milliseconds={}".format(conf.timeout)) if conf.rev_proxy_disable: self.atstccfgCmd.append("--traffic-ops-disable-proxy") + if not conf.verify: + self.atstccfgCmd.append("--traffic-ops-insecure") def __enter__(self): @@ -238,29 +241,23 @@ def getMyChkconfig(self) -> typing.List[dict]: :raises ConnectionError: when something goes wrong communicating with Traffic Ops """ + logging.info("Fetching chkconfig") - # Ah, read-only properties that gut functionality, my favorite. - tmp = self.api_base_url - self._api_base_url = urljoin(self._server_url, '/').rstrip('/') + '/' - - uri = "ort/%s/chkconfig" % self.hostname - logging.info("Fetching chkconfig from %s", uri) - + ret = None for _ in range(self.retries): try: - r = self.get(uri) - break - except (InvalidJSONError, OperationError, LoginError, RequestException) as e: + proc = subprocess.run(self.atstccfgCmd + ["--get-data=chkconfig"]) + logging.debug("Raw response: %s", proc.stdout) + logging.error(proc.stderr) + if proc.returncode == 0: + ret = json.loads(proc.stdout) + break + except (json.JSONDecodeError, InvalidJSONError, OperationError, LoginError, RequestException) as e: logging.debug("chkconfig fetch failure: %r", e, exc_info=True, stack_info=True) else: - self._api_base_url = tmp raise ConnectionError("Failed to fetch 'chkconfig' from Traffic Ops - connection lost") - self._api_base_url = tmp - - logging.debug("Raw response from Traffic Ops: %s", r[1].text) - - return r[0] + return ret def getMyUpdateStatus(self) -> dict: """ From f5936798b6bd9f00e5e454165c2e2680aac0e6fa Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 12:51:55 -0600 Subject: [PATCH 08/49] Switched statuses fetch to use atstccfg --- .../ort/traffic_ops_ort/main_routines.py | 4 ++-- .../ort/traffic_ops_ort/to_api.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py index b2fa53cdb9..5d48d65948 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py @@ -102,11 +102,11 @@ def deleteOldStatusFiles(myStatus:str, conf:Configuration): doDeleteFiles = conf.mode is not Configuration.Modes.REPORT - for status in conf.api.get_statuses()[0]: + for status in conf.api.get_statuses(): # Only the status name matters try: - status = status.name + status = status["name"] except KeyError as e: logging.debug("Bad status object: %r", status) raise ConnectionError from e diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 53bf463e04..de658053bf 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -125,6 +125,26 @@ def getRaw(self, path:str) -> str: return r.text + def get_statuses(self) -> typing.List[dict]: + """ + Retrieves all statuses from the Traffic Ops instance - using atstccfg. + + :returns: Representations of status objects + :raises: ConnectionError if fetching the statuses fails for any reason + """ + for _ in range(self.retries): + try: + proc = subprocess.run(self.atstccfgCmd + ["--get-data=statuses"]) + logging.debug("Raw response: %s", proc.stdout) + if proc.stderr: + logging.error(proc.stderr) + if proc.returncode == 0: + return json.loads(proc.stdout) + except (subprocess.SubprocessError, OSError, json.JSONDecodeError) as e: + logging.error("status fetch failure: %s", e) + raise ConnectionError("Failed to fetch statuses from atstccfg") + + def getMyPackages(self) -> typing.List[packaging.Package]: """ Fetches a list of the packages specified by Traffic Ops that should exist on this server. From 5b7eb7d2cdb1f01369b3143551442fe39f900397 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 12:57:36 -0600 Subject: [PATCH 09/49] Fixed update-status fetch function --- .../ort/traffic_ops_ort/main_routines.py | 2 +- .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py index 5d48d65948..3c291ae56e 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py @@ -46,7 +46,7 @@ def syncDSState(conf:Configuration) -> bool: """ logging.info("starting syncDS state fetch") - updateStatus = conf.api.getMyUpdateStatus()[0] + updateStatus = conf.api.getMyUpdateStatus() logging.debug("Retrieved raw update status: %r", updateStatus) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index de658053bf..16b6aab460 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -290,18 +290,14 @@ def getMyUpdateStatus(self) -> dict: for _ in range(self.retries): try: proc = subprocess.run(self.atstccfgCmd + ["--get-data=update-status"]) - logging.info(proc.stdout) + logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: - break; - except (InvalidJSONError, LoginError, OperationError, RequestException) as e: + return json.loads(proc.stdout) + except (subprocess.SubprocessError, OSError, json.JSONDecodeError) as e: logging.debug("update status fetch failure: %r", e, exc_info=True, stack_info=True) - else: - raise ConnectionError("Failed to fetch update status - connection was lost") - - logging.debug("Raw response from Traffic Ops: %s", r[1].text) - return r[0] + raise ConnectionError("Failed to fetch update status - connection was lost") def getMyStatus(self) -> str: """ From 032893605dbf6a495ca77841f283290b9cc2d6f8 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 12:59:35 -0600 Subject: [PATCH 10/49] Logical consistency update --- .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 16b6aab460..f12d78615e 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -263,21 +263,17 @@ def getMyChkconfig(self) -> typing.List[dict]: logging.info("Fetching chkconfig") - ret = None for _ in range(self.retries): try: proc = subprocess.run(self.atstccfgCmd + ["--get-data=chkconfig"]) logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: - ret = json.loads(proc.stdout) - break - except (json.JSONDecodeError, InvalidJSONError, OperationError, LoginError, RequestException) as e: + return json.loads(proc.stdout) + except (json.JSONDecodeError, OSError, subprocess.SubprocessError) as e: logging.debug("chkconfig fetch failure: %r", e, exc_info=True, stack_info=True) - else: - raise ConnectionError("Failed to fetch 'chkconfig' from Traffic Ops - connection lost") - return ret + raise ConnectionError("Failed to fetch 'chkconfig' from Traffic Ops - connection lost") def getMyUpdateStatus(self) -> dict: """ From ce94b8238fe084c67489dd557e38cfe369511ff1 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 14:57:19 -0600 Subject: [PATCH 11/49] Added method for parsing proprietary multipart responses from atstccfg --- .../cdn-in-a-box/ort/traffic_ops_ort/utils.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py index fa8c75b8b6..4de53edb37 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py @@ -22,6 +22,7 @@ import logging from sys import stderr import requests +import typing def getYesNoResponse(prmpt:str, default:str = None) -> bool: """ @@ -96,3 +97,50 @@ def getJSONResponse(uri:str, cookies:dict = None, verify:bool = True) -> dict: logging.debug("Response: %r\n%r", response.headers, response.content) return response.json() + +def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: + """ + Parses a multipart/mixed-type payload and returns each contiguous chunk. + + :param raw: The raw payload - without any HTTP status line. + :returns: A list where each element is a tuple where the first element is a chunk of the message. All headers are discarded except 'Path', which is the second element of each tuple if it was found in the chunk. + :raises: ValueError if the raw payload cannot be parsed as a multipart/mixed-type message. + """ + try: + hdr_index = raw.index("\r\n\r\n") + headers = {line.split(':')[0].casefold(): line.split(':')[1] for line in raw[:hdr_index].splitlines()} + except (IndexError, ValueError) as e: + raise ValueError("Invalid or corrupt multipart header") from e + + ctype = headers.get("content-type") + if not ctype: + raise ValueError("Message is missing 'Content-Type' header") + + try: + param_index = ctype.index(";") + params = {param.split('=')[0].trim(): param.split('=')[1].trim() for param in ctype[param_index+1:].split(';')} + except (IndexError, ValueError) as e: + raise ValueError("Invalid or corrupt 'Content-Type' header") from e + + boundary = params.get("boundary") + if not boundary: + raise ValueError("'Content-Type' header missing 'boundary' parameter") + + chunks = raw.split(f"--{boundary}")[1:] #ignore prologue + if chunks[-1].trim() != "--": + logging.warning("Final chunk appears invalid - possible bad message payload") + else: + chunks = chunks[:-1] + + ret = [] + for i, chunk in enumerate(chunks): + try: + hdr_index = chunk.index("\r\n\r\n") + headers = {line.split(':')[0].casefold(): line.split(':')[1] for line in chunk[:hdr_index].splitlines()} + except (IndexError, ValueError) as e: + logging.debug("chunk: %s", chunk) + raise ValueError(f"Chunk #{i} poorly formed") + + ret.append((chunk[hdr_index+4:].replace("\r",""), headers.get("path"))) + + return ret From a2862298df01f3229248577af6d1eb2fd7c76857 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:00:54 -0600 Subject: [PATCH 12/49] Added direct construction method to ConfigFile objects --- .../ort/traffic_ops_ort/config_files.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index 9b3a89cdeb..3918df020d 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -60,13 +60,15 @@ class ConfigFile(): contents = "" #: The full contents of the file - as configured in TO, not the on-disk contents sanitizedContents = "" #: Will store the contents after sanitization - def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/"): + def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/", *unused_args, contents: str = None, path: str = None): """ Constructs a :class:`ConfigFile` object from a raw API response :param raw: A raw config file from an API response :param toURL: The URL of a valid Traffic Ops host :param tsroot: The absolute path to the root of an Apache Traffic Server installation + :param contents: Directly constructs a ConfigFile from the passed contents. Must be used with path, and causes raw to be ignored. + :param path: Sets the full path to the file when constructing ConfigFiles directly from contents. :raises ValueError: if ``raw`` does not faithfully represent a configuration file >>> a = ConfigFile({"fnameOnDisk": "test", @@ -77,7 +79,17 @@ def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/"): ConfigFile(path='/path/to/test', URI='http://example.com/test', scope='servers') >>> a.SSLdir "/etc/trafficserver/ssl" + >>> ConfigFile(contents='testquest\n', path='/path/to/test') + ConfigFile(path='/path/to/test', URI='', scope='') """ + self.SSLdir = os.path.join(tsroot, "etc", "trafficserver", "ssl") + + if contents is not None: + if path is None: + raise ValueError("cannot construct from direct contents without setting path") + self.fname, self.location = os.path.split(path) + self.contents = contents + return if raw is not None: try: self.fname = raw["fnameOnDisk"] @@ -90,7 +102,6 @@ def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/"): except (KeyError, TypeError, IndexError) as e: raise ValueError from e - self.SSLdir = os.path.join(tsroot, "etc", "trafficserver", "ssl") def __repr__(self) -> str: """ @@ -175,8 +186,10 @@ def update(self, conf:Configuration) -> bool: if not self.contents: self.fetchContents(conf.api) finalContents = sanitizeContents(self.contents, conf) - else: + else if self.URI: finalContents = self.contents + else: + finalContents = sanitizeContents(self.contents, conf) # Ensure POSIX-compliant files if not finalContents.endswith('\n'): @@ -351,11 +364,8 @@ def sanitizeContents(raw:str, conf:Configuration) -> str: """ out = [] - # These double curly braces escape the behaviour of Python's `str.format` method to attempt - # to use curly brace-enclosed text as a key into a dictonary of its arguments. They'll be - # rendered into single braces in the output of `.format`, leaving the string ultimately - # unchanged in that respect. - for line in conf.serverInfo.sanitize(raw, conf.hostname).splitlines(): + lines = (conf.servicerInfo.sanitize(raw, conf.hostname) if conf.servicerInfo else raw).splitlines() + for line in lines: tmp=(" ".join(line.split())).strip() #squeezes spaces and trims leading and trailing spaces tmp=tmp.replace("&", '&') #decodes HTML-encoded ampersands tmp=tmp.replace(">", '>') #decodes HTML-encoded greater-than symbols From 485c33b25df07fe13625fe48f5ea107fca6d5ce5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:01:54 -0600 Subject: [PATCH 13/49] Fixed incorrect revalidation status checking --- .../cdn-in-a-box/ort/traffic_ops_ort/main_routines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py index 3c291ae56e..1fc4653d35 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py @@ -78,7 +78,7 @@ def revalidateState(conf:Configuration) -> bool: """ logging.info("starting revalidation state fetch") - updateStatus = conf.api.getMyUpdateStatus()[0] + updateStatus = conf.api.getMyUpdateStatus() logging.debug("Retrieved raw revalidation status: %r", updateStatus) if (conf.wait_for_parents and From 62d5d4dedd3c73ddca7e10d81d6edd6cf7cb2bbe Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:07:18 -0600 Subject: [PATCH 14/49] Updated configuration file fetch to use atstccfg --- .../ort/traffic_ops_ort/main_routines.py | 8 ---- .../ort/traffic_ops_ort/to_api.py | 48 ++++++++++--------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py index 1fc4653d35..4996ef6c9e 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py @@ -259,10 +259,6 @@ def processConfigurationFiles(conf:Configuration) -> bool: for file in myFiles: try: - file = config_files.ConfigFile(file, conf.TOURL) - if conf.mode is conf.Modes.REVALIDATE and file.fname != "regex_revalidate.config": - logging.info("Skipping file %s because is not a revalidation file", file.fname) - continue logging.info("\n============ Processing File: %s ============", file.fname) if file.update(conf) and file.fname in services.FILES_THAT_REQUIRE_RELOADS: services.NEEDED_RELOADS.add(services.FILES_THAT_REQUIRE_RELOADS[file.fname]) @@ -275,10 +271,6 @@ def processConfigurationFiles(conf:Configuration) -> bool: logging.error("An error occurred while trying to update %s", file.fname) logging.debug("%s", e, exc_info=True, stack_info=True) return False - except ValueError as e: - logging.error("%s does not appear to be a valid 'configfile' object, or has invalid contents!") - logging.debug("%s", e, exc_info=True, stack_info=True) - return False return True diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index f12d78615e..8286c0eb3a 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -33,7 +33,8 @@ from trafficops.tosession import TOSession from trafficops.restapi import LoginError, OperationError, InvalidJSONError -from . import packaging +from . import packaging, utils +from .config_files import ConfigFile from .configuration import Configuration class API(TOSession): @@ -184,7 +185,7 @@ def setConfigFileAPIVersion(self, files: Munch) -> None: for configFile in files.configFiles: configFile.apiUri = match_api_base.sub(api_base_replacement, configFile.apiUri) - def getMyConfigFiles(self, conf:Configuration) -> typing.List[dict]: + def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: """ Fetches configuration files constructed by Traffic Ops for this server @@ -197,31 +198,32 @@ def getMyConfigFiles(self, conf:Configuration) -> typing.List[dict]: :raises ConnectionError: when something goes wrong communicating with Traffic Ops """ logging.info("Fetching list of configuration files from Traffic Ops") + atstccfg_cmd = self.atstccfgCmd + if conf.mode is Configuration.Modes.REVALIDATE: + atstccfg_cmd = self.atstccfg_cmd + ["--revalidate-only"] for _ in range(self.retries): try: - # The API function decorator confuses pylint into thinking this doesn't return - #pylint: disable=E1111 - myFiles = self.get_server_config_files(host_name=self.hostname) - #pylint: enable=E1111 - break - except (InvalidJSONError, LoginError, OperationError, RequestException) as e: + proc = subprocess.run(atstccfg_cmd) + logging.debug("Raw response: %s", proc.stdout) + if proc.stderr: + logging.error(proc.stderr) + if proc.returncode == 0: + return [ConfigFile(tsroot=conf.tsroot, contents=x[0], path=x[1]) for x in utils.parse_multipart(proc.stdout)] + except (subprocess.SubprocessError, ValueError, OSError) as e: logging.debug("config file fetch failure: %r", e, exc_info=True, stack_info=True) - else: - raise ConnectionError("Failed to fetch configuration files from Traffic Ops") - logging.debug("Raw response from Traffic Ops: %s", myFiles[1].text) - myFiles = myFiles[0] - self.setConfigFileAPIVersion(myFiles) - - try: - conf.serverInfo = ServerInfo(myFiles.info) - # if there's a reverse proxy, switch urls. - if conf.serverInfo.toRevProxyUrl and not conf.rev_proxy_disable: - self._server_url = conf.serverInfo.toRevProxyUrl - self._api_base_url = urljoin(self._server_url, '/api/%s' % self.VERSION).rstrip('/') + '/' - return myFiles.configFiles - except (KeyError, AttributeError, ValueError) as e: - raise ConnectionError("Malformed response from Traffic Ops to update status request!") from e + raise ConnectionError("Failed to fetch configuration files from Traffic Ops") + + # Server-info sanitization is now done by atstccfg + # try: + # conf.serverInfo = ServerInfo(myFiles.info) + # # if there's a reverse proxy, switch urls. + # if conf.serverInfo.toRevProxyUrl and not conf.rev_proxy_disable: + # self._server_url = conf.serverInfo.toRevProxyUrl + # self._api_base_url = urljoin(self._server_url, '/api/%s' % self.VERSION).rstrip('/') + '/' + # return myFiles.configFiles + # except (KeyError, AttributeError, ValueError) as e: + # raise ConnectionError("Malformed response from Traffic Ops to update status request!") from e def updateTrafficOps(self, mode:Configuration.Modes): """ From 0de93d65b825c60e39d2fa7171cc746689903381 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:15:27 -0600 Subject: [PATCH 15/49] Switched to atstccfg for package fetch --- .../ort/traffic_ops_ort/to_api.py | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 8286c0eb3a..ce4bc4d0fc 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -155,29 +155,19 @@ def getMyPackages(self) -> typing.List[packaging.Package]: """ logging.info("Fetching this server's package list from Traffic Ops") - # Ah, read-only properties that gut functionality, my favorite. - tmp = self.api_base_url - self._api_base_url = urljoin(self._server_url, '/').rstrip('/') + '/' - - packagesPath = '/'.join(("ort", self.hostname, "packages")) + atstccfg_cmd = self.atstccfg_cmd + ["--get-data=packages"] for _ in range(self.retries): try: - myPackages = self.get(packagesPath) - break - except (LoginError, OperationError, InvalidJSONError, RequestException) as e: + proc = subprocess.run(atstccfg_cmd) + logging.debug("Raw output: %s", proc.stdout) + if proc.stderr: + logging.error("proc.stderr") + if proc.returncode == 0: + return [Package(p) for p in json.loads(proc.stdout)] + except (ValueError, IndexError, json.JSONDecodeError, OSError, subprocess.SubprocessError) as e: logging.debug("package fetch failure: %r", e, stack_info=True, exc_info=True) - else: - self._api_base_url = tmp - raise ConnectionError("Failed to get a response for packages") - - self._api_base_url = tmp - logging.debug("Raw package response: %s", myPackages[1].text) - - try: - return [packaging.Package(p) for p in myPackages[0]] - except ValueError: - raise ConnectionError + raise ConnectionError("Failed to get a response for packages") def setConfigFileAPIVersion(self, files: Munch) -> None: match_api_base = re.compile(r'^(/api/)\d+\.\d+(/)') From f25822b16ae3d7003162a460b1b49f0f6b543ca7 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:17:06 -0600 Subject: [PATCH 16/49] Removed now-unused method --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index ce4bc4d0fc..1b3d946b89 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -169,12 +169,6 @@ def getMyPackages(self) -> typing.List[packaging.Package]: raise ConnectionError("Failed to get a response for packages") - def setConfigFileAPIVersion(self, files: Munch) -> None: - match_api_base = re.compile(r'^(/api/)\d+\.\d+(/)') - api_base_replacement = r'\g<1>%s\2' % API.VERSION - for configFile in files.configFiles: - configFile.apiUri = match_api_base.sub(api_base_replacement, configFile.apiUri) - def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: """ Fetches configuration files constructed by Traffic Ops for this server From 8c40899dc03a97e00109ddc7d6a7d7777841ce43 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:19:05 -0600 Subject: [PATCH 17/49] Stopped using deprecated API version --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 1b3d946b89..bc5ad764fb 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -41,13 +41,11 @@ class API(TOSession): """ This class extends :class:`trafficops.tosession.TOSession` to provide some ease-of-use functionality for getting things needed by :term:`ORT`. - - TODO: update to 2.0 if/when atstccfg support is integrated. """ #: This should always be the latest API version supported - note this breaks compatability with #: older ATC versions. Go figure. - VERSION = "1.4" + VERSION = "2.0" def __init__(self, conf:Configuration): """ @@ -145,7 +143,6 @@ def get_statuses(self) -> typing.List[dict]: logging.error("status fetch failure: %s", e) raise ConnectionError("Failed to fetch statuses from atstccfg") - def getMyPackages(self) -> typing.List[packaging.Package]: """ Fetches a list of the packages specified by Traffic Ops that should exist on this server. From 10a253ad5f3831bccae67b789b9df22b0d55ef5f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:35:10 -0600 Subject: [PATCH 18/49] Fixed syntax error --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index 3918df020d..15b491dda0 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -186,7 +186,7 @@ def update(self, conf:Configuration) -> bool: if not self.contents: self.fetchContents(conf.api) finalContents = sanitizeContents(self.contents, conf) - else if self.URI: + elif self.URI: finalContents = self.contents else: finalContents = sanitizeContents(self.contents, conf) From 3112c6c1c1d3921ce34176eabf8c123214e32bf5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:38:33 -0600 Subject: [PATCH 19/49] Added ort.py log output to cache container logs --- infrastructure/cdn-in-a-box/edge/run.sh | 2 +- infrastructure/cdn-in-a-box/mid/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/cdn-in-a-box/edge/run.sh b/infrastructure/cdn-in-a-box/edge/run.sh index 72ec22e70e..80701d350d 100755 --- a/infrastructure/cdn-in-a-box/edge/run.sh +++ b/infrastructure/cdn-in-a-box/edge/run.sh @@ -87,4 +87,4 @@ until grep -q demo1 /etc/trafficserver/remap.config; do done touch /var/log/trafficserver/diags.log -tail -Fn +1 /var/log/trafficserver/diags.log +tail -Fn +1 /var/log/trafficserver/diags.log /var/log/ort.log diff --git a/infrastructure/cdn-in-a-box/mid/run.sh b/infrastructure/cdn-in-a-box/mid/run.sh index 9154e8ee1c..fb9ab49c09 100755 --- a/infrastructure/cdn-in-a-box/mid/run.sh +++ b/infrastructure/cdn-in-a-box/mid/run.sh @@ -77,4 +77,4 @@ chmod "0644" "/etc/cron.d/traffic_ops_ort-cron" && crontab "/etc/cron.d/traffic_ crond -im off touch /var/log/trafficserver/diags.log -tail -Fn +1 /var/log/trafficserver/diags.log +tail -Fn +1 /var/log/trafficserver/diags.log /var/log/ort.log From 1bf31bc1a1bb002d8c66ba68901b10a5abf73228 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 15:52:44 -0600 Subject: [PATCH 20/49] Fixed output capture in subprocesses --- .../cdn-in-a-box/ort/traffic_ops_ort/__init__.py | 4 ++++ .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py index 5e8f425fd7..4b3e94c3db 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py @@ -339,6 +339,7 @@ import argparse import datetime +from distutils.spawn import find_executable import logging import os import random @@ -538,5 +539,8 @@ def main() -> int: print("(Hint: use -h/--help for usage)", file=sys.stderr) return 1 + if not find_executable("atstccfg"): + print("Could not find atstccfg executable - this is required to run ORT!", file=sys.stderr) + return 1 return doMain(args) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index bc5ad764fb..922e83b751 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -133,7 +133,7 @@ def get_statuses(self) -> typing.List[dict]: """ for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=statuses"]) + proc = subprocess.run(self.atstccfgCmd + ["--get-data=statuses"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) if proc.stderr: logging.error(proc.stderr) @@ -155,7 +155,7 @@ def getMyPackages(self) -> typing.List[packaging.Package]: atstccfg_cmd = self.atstccfg_cmd + ["--get-data=packages"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfg_cmd) + proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw output: %s", proc.stdout) if proc.stderr: logging.error("proc.stderr") @@ -184,7 +184,7 @@ def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: atstccfg_cmd = self.atstccfg_cmd + ["--revalidate-only"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfg_cmd) + proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) if proc.stderr: logging.error(proc.stderr) @@ -226,7 +226,7 @@ def updateTrafficOps(self, mode:Configuration.Modes): atstccfgCmd = self.atstccfgCmd + ["--set-queue-status=false", "--set-reval-status=false"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfgCmd, capture_output=True, text=True) + proc = subprocess.run(atstccfgCmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.info(proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: @@ -248,7 +248,7 @@ def getMyChkconfig(self) -> typing.List[dict]: for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=chkconfig"]) + proc = subprocess.run(self.atstccfgCmd + ["--get-data=chkconfig"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: @@ -268,7 +268,7 @@ def getMyUpdateStatus(self) -> dict: logging.info("Fetching update status from Traffic Ops") for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=update-status"]) + proc = subprocess.run(self.atstccfgCmd + ["--get-data=update-status"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: From ce581f560a4b76bf92a9e4947df7cb69659867be Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 16:17:42 -0600 Subject: [PATCH 21/49] Fixed incorrect formatting of cache-name option-argument --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 922e83b751..e16e1d6141 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -75,7 +75,7 @@ def __init__(self, conf:Configuration): self.atstccfgCmd = [ "atstccfg", "--traffic-ops-url=http{}://{}:{}".format("s" if conf.useSSL else "", conf.toHost, conf.toPort), - "--cache-host-name={}".format(conf.hostname), + "--cache-host-name={}".format(self.hostname), "--traffic-ops-user={}".format(conf.username), "--traffic-ops-password={}".format(conf.password), "--log-location-error=stderr", From 4307713e130e41c249096d0264200c4e454e2ecb Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 16:22:12 -0600 Subject: [PATCH 22/49] linting fix --- .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index e16e1d6141..4091c5f207 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -72,7 +72,7 @@ def __init__(self, conf:Configuration): self.hostname = conf.shortHostname - self.atstccfgCmd = [ + self.atstccfg_cmd = [ "atstccfg", "--traffic-ops-url=http{}://{}:{}".format("s" if conf.useSSL else "", conf.toHost, conf.toPort), "--cache-host-name={}".format(self.hostname), @@ -84,11 +84,11 @@ def __init__(self, conf:Configuration): ] if conf.timeout is not None and conf.timeout >= 0: - self.atstccfgCmd.append("--traffic-ops-timeout-milliseconds={}".format(conf.timeout)) + self.atstccfg_cmd.append("--traffic-ops-timeout-milliseconds={}".format(conf.timeout)) if conf.rev_proxy_disable: - self.atstccfgCmd.append("--traffic-ops-disable-proxy") + self.atstccfg_cmd.append("--traffic-ops-disable-proxy") if not conf.verify: - self.atstccfgCmd.append("--traffic-ops-insecure") + self.atstccfg_cmd.append("--traffic-ops-insecure") def __enter__(self): @@ -133,7 +133,7 @@ def get_statuses(self) -> typing.List[dict]: """ for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=statuses"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=statuses"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) if proc.stderr: logging.error(proc.stderr) @@ -179,7 +179,7 @@ def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: :raises ConnectionError: when something goes wrong communicating with Traffic Ops """ logging.info("Fetching list of configuration files from Traffic Ops") - atstccfg_cmd = self.atstccfgCmd + atstccfg_cmd = self.atstccfg_cmd if conf.mode is Configuration.Modes.REVALIDATE: atstccfg_cmd = self.atstccfg_cmd + ["--revalidate-only"] for _ in range(self.retries): @@ -223,7 +223,7 @@ def updateTrafficOps(self, mode:Configuration.Modes): if mode is Configuration.Modes.REPORT: return - atstccfgCmd = self.atstccfgCmd + ["--set-queue-status=false", "--set-reval-status=false"] + atstccfgCmd = self.atstccfg_cmd + ["--set-queue-status=false", "--set-reval-status=false"] for _ in range(self.retries): try: proc = subprocess.run(atstccfgCmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) @@ -248,7 +248,7 @@ def getMyChkconfig(self) -> typing.List[dict]: for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=chkconfig"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=chkconfig"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: @@ -268,7 +268,7 @@ def getMyUpdateStatus(self) -> dict: logging.info("Fetching update status from Traffic Ops") for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfgCmd + ["--get-data=update-status"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=update-status"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) logging.debug("Raw response: %s", proc.stdout) logging.error(proc.stderr) if proc.returncode == 0: From 0810fab862c655b3969ea6d2ad739236aa1c9318 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 16:33:15 -0600 Subject: [PATCH 23/49] Removed now-unused tests --- infrastructure/cdn-in-a-box/cache/Dockerfile | 7 +++-- .../cdn-in-a-box/ort/tests/__init__.py | 16 ---------- .../ort/tests/test_api_version_replacement.py | 30 ------------------- 3 files changed, 4 insertions(+), 49 deletions(-) delete mode 100644 infrastructure/cdn-in-a-box/ort/tests/__init__.py delete mode 100644 infrastructure/cdn-in-a-box/ort/tests/test_api_version_replacement.py diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index d5aae9af0b..5fb99f9303 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -48,11 +48,12 @@ RUN mkdir -p /var/trafficserver /opt/ort && \ RUN setcap CAP_NET_BIND_SERVICE=+eip /bin/traffic_server && setcap CAP_NET_BIND_SERVICE=+eip /bin/traffic_manager && setcap CAP_NET_BIND_SERVICE=+eip /bin/trafficserver && setcap CAP_NET_BIND_SERVICE=+eip /bin/traffic_cop -ADD infrastructure/cdn-in-a-box/ort /opt/ort/ - WORKDIR /opt -RUN touch /var/log/ort.log && pip3 install Apache-TrafficControl==1.1.3 && pip3 install ./ort && cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && mkdir -p /opt/init.d && cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort +ADD infrastructure/cdn-in-a-box/ort /opt/ort/ +ADD traffic_control/clients/python /opt/Apache-TrafficControl/ + +RUN touch /var/log/ort.log && pip3 install ./Apache-TrafficControl && pip3 install ./ort && cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && mkdir -p /opt/init.d && cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / ADD infrastructure/cdn-in-a-box/cache/atstccfg /bin/ diff --git a/infrastructure/cdn-in-a-box/ort/tests/__init__.py b/infrastructure/cdn-in-a-box/ort/tests/__init__.py deleted file mode 100644 index 13a83393a9..0000000000 --- a/infrastructure/cdn-in-a-box/ort/tests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/infrastructure/cdn-in-a-box/ort/tests/test_api_version_replacement.py b/infrastructure/cdn-in-a-box/ort/tests/test_api_version_replacement.py deleted file mode 100644 index 884fcf9f15..0000000000 --- a/infrastructure/cdn-in-a-box/ort/tests/test_api_version_replacement.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from unittest import TestCase -from traffic_ops_ort.to_api import API -from munch import Munch - - -class ApiVersionTest(TestCase): - def test_api_version_changed(self) -> None: - changed = Munch({'configFiles': [Munch({'key': 'value', - 'apiUri': '/api/3.85/endpoint'})]}) - expected = Munch({'configFiles': [Munch({'key': 'value', - 'apiUri': '/api/%s/endpoint' % API.VERSION})]}) - API.setConfigFileAPIVersion(None, changed) - self.assertEqual(expected, changed) From f592ef850656fb87b0fbae1009e5d88a3950fb84 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:15:17 -0600 Subject: [PATCH 24/49] Added test runner --- .../cdn-in-a-box/ort/doctest-runner.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 infrastructure/cdn-in-a-box/ort/doctest-runner.py diff --git a/infrastructure/cdn-in-a-box/ort/doctest-runner.py b/infrastructure/cdn-in-a-box/ort/doctest-runner.py new file mode 100755 index 0000000000..0ceb9bd617 --- /dev/null +++ b/infrastructure/cdn-in-a-box/ort/doctest-runner.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if __name__ == "__main__": + from traffic_ops_ort import config_files, configuration, main_routines, packaging, services, to_api, utils + import doctest + doctest.testmod(config_files) + doctest.testmod(configuration) + doctest.testmod(main_routines) + doctest.testmod(packaging) + doctest.testmod(services) + doctest.testmod(to_api) + doctest.testmod(utils) From aeec559f32cdf6b7af4d7a5efa72c67a21a9f04c Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:15:34 -0600 Subject: [PATCH 25/49] Fixed testing errors and errors that tests found --- .../ort/traffic_ops_ort/config_files.py | 15 +++---- .../cdn-in-a-box/ort/traffic_ops_ort/utils.py | 42 ++++++++++++++++--- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index 15b491dda0..b8b77afafb 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -73,22 +73,23 @@ def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/", *unused_ar >>> a = ConfigFile({"fnameOnDisk": "test", ... "location": "/path/to", - ... "apiURI":"/test", - ... "scope": servers}, "http://example.com/") + ... "apiUri":"/test", + ... "scope": "servers"}, "http://example.com/") >>> a ConfigFile(path='/path/to/test', URI='http://example.com/test', scope='servers') >>> a.SSLdir - "/etc/trafficserver/ssl" - >>> ConfigFile(contents='testquest\n', path='/path/to/test') - ConfigFile(path='/path/to/test', URI='', scope='') + '/etc/trafficserver/ssl' + >>> ConfigFile(contents='testquest', path='/path/to/test') + ConfigFile(path='/path/to/test', URI=None, scope=None) """ self.SSLdir = os.path.join(tsroot, "etc", "trafficserver", "ssl") if contents is not None: if path is None: raise ValueError("cannot construct from direct contents without setting path") - self.fname, self.location = os.path.split(path) + self.location, self.fname = os.path.split(path) self.contents = contents + self.scope = None return if raw is not None: try: @@ -109,7 +110,7 @@ def __repr__(self) -> str: >>> repr(ConfigFile({"fnameOnDisk": "test", ... "location": "/path/to", - ... "apiURI": "http://test", + ... "apiUri": "test", ... "scope": "servers"}, "http://example.com/")) "ConfigFile(path='/path/to/test', URI='http://example.com/test', scope='servers')" """ diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py index 4de53edb37..91f7c4ca9d 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py @@ -105,6 +105,38 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: :param raw: The raw payload - without any HTTP status line. :returns: A list where each element is a tuple where the first element is a chunk of the message. All headers are discarded except 'Path', which is the second element of each tuple if it was found in the chunk. :raises: ValueError if the raw payload cannot be parsed as a multipart/mixed-type message. + + >>> testdata = '''MIME-Version: 1.0\\r + ... Content-Type: multipart/mixed; boundary=test\\r + ... \\r + ... --test\\r + ... Content-Type: text/plain; charset=us-ascii\\r + ... Path: /path/to/ats/root/directory/etc/trafficserver/fname\\r + ... \\r + ... # A fake testing file that wasn't generated at all on some date\\r + ... CONFIG proxy.config.way.too.many.period.separated.words INT 1\\r + ... \\r + ... --test\\r + ... Content-Type: text/plain; charset=utf8 + ... Path: /path/to/ats/root/directory/etc/trafficserver/othername\\r + ... \\r + ... # The same header again\\r + ... CONFIG proxy.config.the.same.insane.chain.of.words.again.but.the.last.one.is.different INT 0\\r + ... \\r + ... --test--''' + >>> output = parse_multipart(testdata) + >>> print(output[0][0]) + # A fake testing file that wasn't generated at all on some date + CONFIG proxy.config.way.too.many.period.separated.words INT 1 + + >>> output[0][1] + '/path/to/ats/root/directory/etc/trafficserver/fname' + >>> print(output[1][0]) + # The same header again + CONFIG proxy.config.the.same.insane.chain.of.words.again.but.the.last.one.is.different INT 0 + + >>> output[1][1] + '/path/to/ats/root/directory/etc/trafficserver/othername' """ try: hdr_index = raw.index("\r\n\r\n") @@ -118,7 +150,7 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: try: param_index = ctype.index(";") - params = {param.split('=')[0].trim(): param.split('=')[1].trim() for param in ctype[param_index+1:].split(';')} + params = {param.split('=')[0].strip(): param.split('=')[1].strip() for param in ctype[param_index+1:].split(';')} except (IndexError, ValueError) as e: raise ValueError("Invalid or corrupt 'Content-Type' header") from e @@ -127,7 +159,7 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: raise ValueError("'Content-Type' header missing 'boundary' parameter") chunks = raw.split(f"--{boundary}")[1:] #ignore prologue - if chunks[-1].trim() != "--": + if chunks[-1].strip() != "--": logging.warning("Final chunk appears invalid - possible bad message payload") else: chunks = chunks[:-1] @@ -136,11 +168,11 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: for i, chunk in enumerate(chunks): try: hdr_index = chunk.index("\r\n\r\n") - headers = {line.split(':')[0].casefold(): line.split(':')[1] for line in chunk[:hdr_index].splitlines()} + headers = {line.split(':')[0].casefold(): line.split(':')[1] for line in chunk[:hdr_index].splitlines() if line} except (IndexError, ValueError) as e: logging.debug("chunk: %s", chunk) - raise ValueError(f"Chunk #{i} poorly formed") + raise ValueError(f"Chunk #{i} poorly formed") from e - ret.append((chunk[hdr_index+4:].replace("\r",""), headers.get("path"))) + ret.append((chunk[hdr_index+4:].replace("\r","").strip(), headers.get("path").strip())) return ret From c6b99ab1b6c88f0b351ad9bdf166153bff0b7407 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:27:02 -0600 Subject: [PATCH 26/49] Fixed testing for carriage returns in actual file contents --- .../cdn-in-a-box/ort/traffic_ops_ort/utils.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py index 91f7c4ca9d..8700f3c763 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py @@ -113,17 +113,18 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: ... Content-Type: text/plain; charset=us-ascii\\r ... Path: /path/to/ats/root/directory/etc/trafficserver/fname\\r ... \\r - ... # A fake testing file that wasn't generated at all on some date\\r - ... CONFIG proxy.config.way.too.many.period.separated.words INT 1\\r - ... \\r + ... # A fake testing file that wasn't generated at all on some date + ... CONFIG proxy.config.way.too.many.period.separated.words INT 1 + ... ... --test\\r - ... Content-Type: text/plain; charset=utf8 + ... Content-Type: text/plain; charset=utf8\\r ... Path: /path/to/ats/root/directory/etc/trafficserver/othername\\r ... \\r - ... # The same header again\\r - ... CONFIG proxy.config.the.same.insane.chain.of.words.again.but.the.last.one.is.different INT 0\\r - ... \\r - ... --test--''' + ... # The same header again + ... CONFIG proxy.config.the.same.insane.chain.of.words.again.but.the.last.one.is.different INT 0 + ... + ... --test--\\r + ... ''' >>> output = parse_multipart(testdata) >>> print(output[0][0]) # A fake testing file that wasn't generated at all on some date From 3ecf7831305c2637c4ba2ad8eac2afaa7e64282c Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:45:55 -0600 Subject: [PATCH 27/49] Fixed encoding handling - is not unicode --- .../ort/traffic_ops_ort/to_api.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 4091c5f207..4bfdd67a54 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -133,12 +133,12 @@ def get_statuses(self) -> typing.List[dict]: """ for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfg_cmd + ["--get-data=statuses"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.debug("Raw response: %s", proc.stdout) - if proc.stderr: - logging.error(proc.stderr) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=statuses"], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.debug("Raw response: %s", proc.stdout.decode()) + if proc.stderr.decode(): + logging.error(proc.stderr.decode()) if proc.returncode == 0: - return json.loads(proc.stdout) + return json.loads(proc.stdout.decode()) except (subprocess.SubprocessError, OSError, json.JSONDecodeError) as e: logging.error("status fetch failure: %s", e) raise ConnectionError("Failed to fetch statuses from atstccfg") @@ -155,12 +155,12 @@ def getMyPackages(self) -> typing.List[packaging.Package]: atstccfg_cmd = self.atstccfg_cmd + ["--get-data=packages"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.debug("Raw output: %s", proc.stdout) - if proc.stderr: - logging.error("proc.stderr") + proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.debug("Raw output: %s", proc.stdout.decode()) + if proc.stderr.decode(): + logging.error("proc.stderr.decode()") if proc.returncode == 0: - return [Package(p) for p in json.loads(proc.stdout)] + return [Package(p) for p in json.loads(proc.stdout.decode())] except (ValueError, IndexError, json.JSONDecodeError, OSError, subprocess.SubprocessError) as e: logging.debug("package fetch failure: %r", e, stack_info=True, exc_info=True) @@ -184,12 +184,12 @@ def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: atstccfg_cmd = self.atstccfg_cmd + ["--revalidate-only"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.debug("Raw response: %s", proc.stdout) - if proc.stderr: - logging.error(proc.stderr) + proc = subprocess.run(atstccfg_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.debug("Raw response: %s", proc.stdout.decode()) + if proc.stderr.decode(): + logging.error(proc.stderr.decode()) if proc.returncode == 0: - return [ConfigFile(tsroot=conf.tsroot, contents=x[0], path=x[1]) for x in utils.parse_multipart(proc.stdout)] + return [ConfigFile(tsroot=conf.tsroot, contents=x[0], path=x[1]) for x in utils.parse_multipart(proc.stdout.decode())] except (subprocess.SubprocessError, ValueError, OSError) as e: logging.debug("config file fetch failure: %r", e, exc_info=True, stack_info=True) @@ -226,9 +226,9 @@ def updateTrafficOps(self, mode:Configuration.Modes): atstccfgCmd = self.atstccfg_cmd + ["--set-queue-status=false", "--set-reval-status=false"] for _ in range(self.retries): try: - proc = subprocess.run(atstccfgCmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.info(proc.stdout) - logging.error(proc.stderr) + proc = subprocess.run(atstccfgCmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.info(proc.stdout.decode()) + logging.error(proc.stderr.decode()) if proc.returncode == 0: break except (LoginError, InvalidJSONError, OperationError, RequestException) as e: @@ -248,11 +248,11 @@ def getMyChkconfig(self) -> typing.List[dict]: for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfg_cmd + ["--get-data=chkconfig"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.debug("Raw response: %s", proc.stdout) - logging.error(proc.stderr) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=chkconfig"], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.debug("Raw response: %s", proc.stdout.decode()) + logging.error(proc.stderr.decode()) if proc.returncode == 0: - return json.loads(proc.stdout) + return json.loads(proc.stdout.decode()) except (json.JSONDecodeError, OSError, subprocess.SubprocessError) as e: logging.debug("chkconfig fetch failure: %r", e, exc_info=True, stack_info=True) @@ -268,11 +268,11 @@ def getMyUpdateStatus(self) -> dict: logging.info("Fetching update status from Traffic Ops") for _ in range(self.retries): try: - proc = subprocess.run(self.atstccfg_cmd + ["--get-data=update-status"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - logging.debug("Raw response: %s", proc.stdout) - logging.error(proc.stderr) + proc = subprocess.run(self.atstccfg_cmd + ["--get-data=update-status"], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + logging.debug("Raw response: %s", proc.stdout.decode()) + logging.error(proc.stderr.decode()) if proc.returncode == 0: - return json.loads(proc.stdout) + return json.loads(proc.stdout.decode()) except (subprocess.SubprocessError, OSError, json.JSONDecodeError) as e: logging.debug("update status fetch failure: %r", e, exc_info=True, stack_info=True) From 9999a085f49bf09dc7a98af6f49184d521bba1d8 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:46:29 -0600 Subject: [PATCH 28/49] Fixed handling of quoted MIME params --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py index 8700f3c763..2243c3e479 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py @@ -107,7 +107,7 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: :raises: ValueError if the raw payload cannot be parsed as a multipart/mixed-type message. >>> testdata = '''MIME-Version: 1.0\\r - ... Content-Type: multipart/mixed; boundary=test\\r + ... Content-Type: multipart/mixed; boundary="test"\\r ... \\r ... --test\\r ... Content-Type: text/plain; charset=us-ascii\\r @@ -155,7 +155,7 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: except (IndexError, ValueError) as e: raise ValueError("Invalid or corrupt 'Content-Type' header") from e - boundary = params.get("boundary") + boundary = params.get("boundary", "").strip('"\'') if not boundary: raise ValueError("'Content-Type' header missing 'boundary' parameter") From 0dcf262fbeb3bf3329a17b86cd5911a6edc6f3e9 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 17:58:28 -0600 Subject: [PATCH 29/49] Fixed typo --- .../cdn-in-a-box/ort/traffic_ops_ort/config_files.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index b8b77afafb..76fd29892e 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -263,10 +263,10 @@ def advancedSSLProcessing(self, conf:Configuration): """ global SSL_KEY_REGEX - logging.info("Doing advanced SSL key processing for CDN '%s'", conf.serverInfo.cdnName) + logging.info("Doing advanced SSL key processing for CDN '%s'", conf.ServerInfo.cdnName) try: - r = conf.api.get_cdn_ssl_keys(cdn_name=conf.serverInfo.cdnName) + r = conf.api.get_cdn_ssl_keys(cdn_name=conf.ServerInfo.cdnName) if r[1].status_code != 200 and r[1].status_code != 204: raise OSError("Bad response code: %d - raw response: %s" % @@ -324,7 +324,7 @@ def advancedSSLProcessing(self, conf:Configuration): break else: logging.critical("Failed to find SSL key in %s for '%s' or by wildcard '%s'!", - conf.serverInfo.cdnName, full, wildcard) + conf.ServerInfo.cdnName, full, wildcard) raise OSError("No cert/key pair for ssl_multicert.config line '%s'" % l) # If even one key was written, we need to make ATS aware of the configuration changes @@ -365,7 +365,7 @@ def sanitizeContents(raw:str, conf:Configuration) -> str: """ out = [] - lines = (conf.servicerInfo.sanitize(raw, conf.hostname) if conf.servicerInfo else raw).splitlines() + lines = (conf.ServerInfo.sanitize(raw, conf.hostname) if conf.ServerInfo else raw).splitlines() for line in lines: tmp=(" ".join(line.split())).strip() #squeezes spaces and trims leading and trailing spaces tmp=tmp.replace("&", '&') #decodes HTML-encoded ampersands From ffaa46beeaffc86726ef5a34ab1ee857d7e9ef6c Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 18:01:52 -0600 Subject: [PATCH 30/49] Skipping advanced SSL processing for directly constructed files --- .../cdn-in-a-box/ort/traffic_ops_ort/config_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index 76fd29892e..d0da7c3ee2 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -249,7 +249,8 @@ def update(self, conf:Configuration) -> bool: logging.info("File doesn't differ from disk; nothing to do") # Now we need to do some advanced processing to a couple specific filenames... unfortunately - if self.fname == "ssl_multicert.config": + # But ONLY if the object wasn't directly constructed. + if self.fname == "ssl_multicert.config" and self.URI: return self.advancedSSLProcessing(conf) or written return written From a75b400e856bff5544102b1f3c3a01ccc5bad10f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 18:51:26 -0600 Subject: [PATCH 31/49] Fixed dependency on old client version --- infrastructure/cdn-in-a-box/ort/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/setup.py b/infrastructure/cdn-in-a-box/ort/setup.py index 2fe5275f64..28e00c9b30 100755 --- a/infrastructure/cdn-in-a-box/ort/setup.py +++ b/infrastructure/cdn-in-a-box/ort/setup.py @@ -60,7 +60,7 @@ ], keywords='network connection configuration TrafficControl', packages=find_packages(exclude=['contrib', 'docs', 'tests']), - install_requires=['setuptools', 'typing', 'requests', 'urllib3', 'distro', 'psutil', 'Apache-TrafficControl==1.1.3'], + install_requires=['setuptools', 'typing', 'requests', 'urllib3', 'distro', 'psutil', 'Apache-TrafficControl'], extras_require={'dev': ['unittest']}, # data_files=[('etc/crontab', ['traffic_ops_ort.crontab'])], entry_points={ From 0e05cf68d8c6aa3fd365350a836e4c7cfe6a3527 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 18:52:35 -0600 Subject: [PATCH 32/49] Incremented version number --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py index 4b3e94c3db..27abc37652 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py @@ -334,7 +334,7 @@ =============== """ -__version__ = "0.1.1" +__version__ = "0.2.0" __author__ = "Brennan Fieck" import argparse From 31ed310eb1cc2aeb610c14d065ff616d04f7b576 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 19:09:05 -0600 Subject: [PATCH 33/49] Moved atstccfg include location to its natural directory --- infrastructure/cdn-in-a-box/.gitignore | 1 - infrastructure/cdn-in-a-box/Makefile | 4 ++-- infrastructure/cdn-in-a-box/cache/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/infrastructure/cdn-in-a-box/.gitignore b/infrastructure/cdn-in-a-box/.gitignore index 774763d981..aea22e038c 100644 --- a/infrastructure/cdn-in-a-box/.gitignore +++ b/infrastructure/cdn-in-a-box/.gitignore @@ -18,4 +18,3 @@ *.rpm traffic_ops/ca optional/vpn/vpnca -cache/atstccfg diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index 13dcba039a..8bb1e8645c 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -53,7 +53,7 @@ TS_SOURCE := $(wildcard ../../traffic_stats/**/*) .PHONY: clean very-clean all nearly-all debug # Default target; builds all pre-requisite rpms from source trees -all: traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm +all: ../../traffic_ops/ort/atstccfg/atstccfg traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm debug: PKG_FLAGS += -d ifneq ($(MAKECMDGOALS), debug) @@ -75,7 +75,7 @@ traffic_router/tomcat.rpm: ../../dist/tomcat-$(SPECIAL_SEASONING) cp -f $? $@ traffic_stats/traffic_stats.rpm: ../../dist/traffic_stats-$(SPECIAL_SAUCE) cp -f $? $@ -cache/atstccfg: $(ORT_SOURCE) +../../traffic_ops/ort/atstccfg/atstccfg: $(ORT_SOURCE) go build -o $@ ../../traffic_ops/ort/atstccfg # Dist rpms diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index 5fb99f9303..8216599683 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -56,5 +56,5 @@ ADD traffic_control/clients/python /opt/Apache-TrafficControl/ RUN touch /var/log/ort.log && pip3 install ./Apache-TrafficControl && pip3 install ./ort && cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && mkdir -p /opt/init.d && cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / -ADD infrastructure/cdn-in-a-box/cache/atstccfg /bin/ +ADD traffic_ops/ort/atstccfg/atstccfg /bin/ CMD /run.sh From 134980a91442613441946bb7439fac2ac4d16701 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 19:51:47 -0600 Subject: [PATCH 34/49] Fixed a typo --- traffic_ops_ort/atstccfg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops_ort/atstccfg/config/config.go b/traffic_ops_ort/atstccfg/config/config.go index 55b6cc50f7..e28b91b365 100644 --- a/traffic_ops_ort/atstccfg/config/config.go +++ b/traffic_ops_ort/atstccfg/config/config.go @@ -96,7 +96,7 @@ func GetCfg() (Cfg, error) { cacheHostNamePtr := flag.StringP("cache-host-name", "n", "", "Host name of the cache to generate config for. Must be the server host name in Traffic Ops, not a URL, and not the FQDN") getDataPtr := flag.StringP("get-data", "d", "", "non-config-file Traffic Ops Data to get. Valid values are update-status, packages, chkconfig, system-info, and statuses") setQueueStatusPtr := flag.StringP("set-queue-status", "q", "", "POSTs to Traffic Ops setting the queue status of the server. Must be 'true' or 'false'. Requires --set-reval-status also be set") - setRevalStatusPtr := flag.StringP("set-reval-status", "a", "", "POSTs to Traffic Ops setting the revaliate status of the server. Must be 'true' or 'false'. Requires --set-queue-status also be set") + setRevalStatusPtr := flag.StringP("set-reval-status", "a", "", "POSTs to Traffic Ops setting the revalidate status of the server. Must be 'true' or 'false'. Requires --set-queue-status also be set") revalOnlyPtr := flag.BoolP("revalidate-only", "y", false, "Whether to exclude files not named 'regex_revalidate.config'") disableProxyPtr := flag.BoolP("traffic-ops-disable-proxy", "p", false, "Whether to not use the Traffic Ops proxy specified in the GLOBAL Parameter tm.rev_proxy.url") From 441172e23259fd4d5b39ad47aac635a8a8458ef8 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 19:58:46 -0600 Subject: [PATCH 35/49] Added support for more standard environment variable --- traffic_ops_ort/atstccfg/config/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/traffic_ops_ort/atstccfg/config/config.go b/traffic_ops_ort/atstccfg/config/config.go index e28b91b365..c2a88a815f 100644 --- a/traffic_ops_ort/atstccfg/config/config.go +++ b/traffic_ops_ort/atstccfg/config/config.go @@ -140,6 +140,9 @@ func GetCfg() (Cfg, error) { if toPass == "" { toPass = os.Getenv("TO_PASS") } + if toPass == "" { + toPass = os.Getenv("TO_PASSWORD") + } usageStr := "Usage: ./" + AppName + " --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache" if strings.TrimSpace(toURL) == "" { From 0ae56114006f0ab5f00eb7b2f01bbb733c4b1f51 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 20:16:39 -0600 Subject: [PATCH 36/49] Fixed incorrect unit label --- traffic_ops_ort/atstccfg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops_ort/atstccfg/config/config.go b/traffic_ops_ort/atstccfg/config/config.go index c2a88a815f..1b107d1b42 100644 --- a/traffic_ops_ort/atstccfg/config/config.go +++ b/traffic_ops_ort/atstccfg/config/config.go @@ -89,7 +89,7 @@ func GetCfg() (Cfg, error) { logLocationWarnPtr := flag.StringP("log-location-warning", "w", "stderr", "Where to log warnings. May be a file path, stdout, stderr, or null.") logLocationInfoPtr := flag.StringP("log-location-info", "i", "stderr", "Where to log information messages. May be a file path, stdout, stderr, or null.") toInsecurePtr := flag.BoolP("traffic-ops-insecure", "s", false, "Whether to ignore HTTPS certificate errors from Traffic Ops. It is HIGHLY RECOMMENDED to never use this in a production environment, but only for debugging.") - toTimeoutMSPtr := flag.IntP("traffic-ops-timeout-milliseconds", "t", 30000, "Timeout in seconds for Traffic Ops requests.") + toTimeoutMSPtr := flag.IntP("traffic-ops-timeout-milliseconds", "t", 30000, "Timeout in milliseconds for Traffic Ops requests.") versionPtr := flag.BoolP("version", "v", false, "Print version information and exit.") listPluginsPtr := flag.BoolP("list-plugins", "l", false, "Print the list of plugins.") helpPtr := flag.BoolP("help", "h", false, "Print usage information and exit") From a7f7ccab150ceb290fa56c7096052744cbc10a6f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 28 Apr 2020 20:26:28 -0600 Subject: [PATCH 37/49] Updated README with new usage information --- traffic_ops_ort/atstccfg/README.md | 91 ++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 18 deletions(-) diff --git a/traffic_ops_ort/atstccfg/README.md b/traffic_ops_ort/atstccfg/README.md index ccecdc0b12..e1011dd5d9 100644 --- a/traffic_ops_ort/atstccfg/README.md +++ b/traffic_ops_ort/atstccfg/README.md @@ -24,25 +24,76 @@ atstccfg is a tool for generating configuration files server-side on ATC cache s ## Usage ``` -atstccfg [-u TO_URL] [-U TO_USER] [-P TO_PASSWORD] [-n] [-r N] [-e ERROR_LOCATION] [-w WARNING_LOCATION] [-i INFO_LOCATION] [-g] [-s] [-t TIMEOUT] [-a MAX_AGE] [-l] +atstccfg [-u TO_URL] [-U TO_USER] [-P TO_PASSWORD] [-n] [-r N] [-e ERROR_LOCATION] [-w WARNING_LOCATION] [-i INFO_LOCATION] [-g] [-s] [-t TIMEOUT] [-a REVAL_STATUS] [-l] ``` The available options are: ``` --a, --cache-file-max-age-seconds Sets the maximum age - in seconds - a cached response can be in order to be considered "fresh" - older files will be re-generated and cached. Default: 60 --e ERROR_LOCATION, --log-location-error ERROR_LOCATION The file location to which to log errors. Respects the special string constants of github.com/apache/trafficcontrol/lib/go-log. Default: 'stderr' --g, --print-generated-files If given, the names of files generated (and not proxied to Traffic Ops) will be printed to stdout, then atstccfg will exit. --h, --help Print usage information and exit. --i INFO_LOCATION, --log-location-info INFO_LOCATION The file location to which to log information messages. Respects the special string constants of github.com/apache/trafficcontrol/lib/go-log. Default: 'stderr' --l, --list-plugins List the loaded plugins and then exit. --n, --no-cache If given, existing cache files will not be used. Cache files will still be created, existing ones just won't be used. --P TO_PASSWORD Authenticate using this password - if not given, atstccfg will attempt to use the value of the TO_PASS environment variable --r N, --num-retries N The number of times to retry getting a file if it fails. Default: 5 --s, --traffic-ops-insecure If given, SSL certificate errors will be ignored when communicating with Traffic Ops. NOT RECOMMENDED FOR PRODUCTION ENVIRONMENTS. --t, --traffic-ops-timeout-milliseconds Sets the timeout - in milliseconds - for requests made to Traffic Ops. Default: 10000 --u TO_URL Request this URL, e.g. 'https://trafficops.infra.ciab.test/servers/edge/configfiles/ats' --U TO_USER Authenticate as the user TO_USER - if not given, atstccfg will attempt to use the value of the TO_USER environment variable --v, --version Print version information and exit. --w WARNING_LOCATION, --log-location-warning WARNING_LOCATION The file location to which to log warnings. Respects the special string constants of github.com/apache/trafficcontrol/lib/go-log. Default: 'stderr' +-a, --set-reval-status string + Sets the reval_pending property of the server in Traffic Ops. Must be 'true' + or 'false'. Requires --set-queue-status also be set. This disables normal + output. +-e, --log-location-error string + A location for error-level logging. Passing "stderr" causes it to log to + STDERR, "stdout" causes logging to STDOUT, "null" disables error-level + logging, and anything else is treated as a path to a file which will contain + the logs. (Default: stderr) +-d, --get-data string + Specifies non-configuration-file data to retrieve from Traffic Ops. This + disables normal output. Valid values are update-status, packages, chkconfig, + system-info, and statuses. Output is in JSON-encoded format. For specifics, + refer to the official documentation. +-h, --help + Print usage information and exit. +-i, --log-location-info string + A location for informative-level logging. Passing "stderr" causes it to log + to STDERR, "stdout" causes logging to STDOUT, "null" disables + informative-level logging, and anything else is treated as a path to a file + which will contain the logs. (Default: stderr) +-l, --list-plugins + Print the list of plug-ins and exit. +-n, --cache-host-name string + Required. Specifies the (short) hostname of the cache server for which + output will be generated. Must be the server host name in Traffic Ops, not a + URL, or Fully Qualified Domain Name. Behavior when more than one server + exists with the passed hostname is undefined. +-p, --traffic-ops-disable-proxy + Bypass the Traffic Ops caching proxy and make requests directly to Traffic + Ops. Has no effect if no such proxy exists. +-P, --traffic-ops-password string + The password to use when authenticating with Traffic Ops password. If not + given, the value of the TO_PASSWORD environment variable is used. If that + environment variable is not set, this option-argument is required. +-q, --set-queue-status string + Sets the upd_pending property of the server in Traffic Ops. Must be 'true' + or 'false'. Requires --set-reval-status also be set. This disables normal + output. +-r, --num-retries int + The number of times to retry getting a file if it fails. (Default 5) +-s, --traffic-ops-insecure + Ignore HTTPS certificate errors from Traffic Ops. It is HIGHLY RECOMMENDED + to never use this in a production environment, but only for debugging. +-t, --traffic-ops-timeout-milliseconds int + Timeout in milliseconds for Traffic Ops requests. (Default 30000) +-u, --traffic-ops-url string + The full URL, including scheme and optionally port number, of the Traffic + Ops server. If not given, the value of the TO_URL environment variable will + be used. If that environment variable is not properly set, this + option-argument is required. +-U, --traffic-ops-user string + The username to use when authenticating with Traffic Ops. If not given, the + value of the TO_USER environment variable will be used. If that environment + variable is not set, this option-argument is required. +-v, --version + Print version information and exit. +-w, --log-location-warning string + A location for warning-level logging. Passing "stderr" causes it to log to + STDERR, "stdout" causes logging to STDOUT, "null" disables warning-level + logging, and anything else is treated as a path to a file which will contain + the logs. (Default: stderr) +-y, --revalidate-only + When given, atstccfg will only emit files relevant for updating content + invalidation jobs. For Apache Traffic Server implementations, this limits + the output to be only files named 'regex_revalidate.config'. ``` atstccfg caches generated files in /tmp/atstccfg_cache/ for re-use. @@ -50,6 +101,10 @@ atstccfg caches generated files in /tmp/atstccfg_cache/ for re-use. ## Updating for new Traffic Control Versions -After a new Traffic Control release, the Traffic Ops client from the new release branch should be vendored at `toreq/vendor`, and all usages of `config.TOClientNew` should be changed to `config.TOClient`. +After a new Traffic Control release, the Traffic Ops client from the new release +branch should be vendored at `toreq/vendor`, and all usages of +`config.TOClientNew` should be changed to `config.TOClient`. -There's a script to do this at `ort/atstccfg/update-to-client/update-to-client.go`. Run the script with no arguments for usage information. +There's a "script" to do this at +[`./update-to-client/update-to-client.go`](./update-to-client). Run the "script" +with no arguments for usage information. From b193725cf316047ac167bc95b247cd1c5eff05f2 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Wed, 29 Apr 2020 10:39:04 -0600 Subject: [PATCH 38/49] Added full usage examples --- traffic_ops_ort/atstccfg/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/traffic_ops_ort/atstccfg/README.md b/traffic_ops_ort/atstccfg/README.md index e1011dd5d9..af13ca4965 100644 --- a/traffic_ops_ort/atstccfg/README.md +++ b/traffic_ops_ort/atstccfg/README.md @@ -24,7 +24,12 @@ atstccfg is a tool for generating configuration files server-side on ATC cache s ## Usage ``` -atstccfg [-u TO_URL] [-U TO_USER] [-P TO_PASSWORD] [-n] [-r N] [-e ERROR_LOCATION] [-w WARNING_LOCATION] [-i INFO_LOCATION] [-g] [-s] [-t TIMEOUT] [-a REVAL_STATUS] [-l] +atstccfg -h +atstccfg -v +atstccfg -l +atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] [-y] -n CACHE_NAME +atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] -n CACHE_NAME -d DATA +atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] -n CACHE_NAME -a REVAL_STATUS -q QUEUE_STATUS ``` The available options are: ``` @@ -93,9 +98,9 @@ The available options are: -y, --revalidate-only When given, atstccfg will only emit files relevant for updating content invalidation jobs. For Apache Traffic Server implementations, this limits - the output to be only files named 'regex_revalidate.config'. + the output to be only files named 'regex_revalidate.config'. Has no effect + if --get-data or --set-queue-status/--set-reval-status is/are used. ``` -atstccfg caches generated files in /tmp/atstccfg_cache/ for re-use. # Development From 3a737a4e690fcba9ee49a25d984cd827d9ca731f Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Wed, 29 Apr 2020 10:41:48 -0600 Subject: [PATCH 39/49] Updated official docs --- docs/source/tools/atstccfg.rst | 67 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/docs/source/tools/atstccfg.rst b/docs/source/tools/atstccfg.rst index 19e66d8b92..dc412481bd 100644 --- a/docs/source/tools/atstccfg.rst +++ b/docs/source/tools/atstccfg.rst @@ -20,7 +20,7 @@ ******** atstccfg ******** -:program:`atstccfg` is a tool for generating configuration files server-side on :abbr:`ATC (Apache Traffic Control)` cache servers. It stores its generated/cached files in ``/tmp/atstccfg_cache/`` for re-use. +:program:`atstccfg` is a tool for generating configuration files server-side on :abbr:`ATC (Apache Traffic Control)` cache servers. .. warning:: :program:`atstccfg` does not have a stable command-line interface, it can and will change without warning. Scripts should avoid calling it for the time being, as its only intended caller is :term:`ORT`. @@ -28,41 +28,76 @@ The source code for :program:`atstccfg` may be found in :atc-file:`traffic_ops/o Usage ===== -``atstccfg [-u TO_URL] [-U TO_USER] [-P TO_PASSWORD] [-n] [-r N] [-e ERROR_LOCATION] [-w WARNING_LOCATION] [-i INFO_LOCATION] [-g] [-s] [-t TIMEOUT] [-a MAX_AGE] [-l] [-h] [-v]`` +- ``atstccfg -h`` +- ``atstccfg -v`` +- ``atstccfg -l`` +- ``atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] [-y] -n CACHE_NAME`` +- ``atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] -n CACHE_NAME -d DATA`` +- ``atstccfg [-e ERROR_LOCATION] [-i INFO_LOCATION] [-p] [-P TO_PASSWORD] [-r N] [-s] [-t TIMEOUT] [-u TO_URL] [-U TO_USER] [-w WARNING_LOCATION] -n CACHE_NAME -a REVAL_STATUS -q QUEUE_STATUS`` + +When called using the fourth form, :program:`atstccfg` provides its normal output. This is the entirety of all configuration files necessary for the server, all provided at once. The output is in :mimetype:`mixed/multipart` format, defined by :rfc:`1521`. Each chunk of the message comes with the proprietary ``Path`` header, which specifies the exact location on disk of the file whose contents are contained in that chunk. Options ------- -.. option:: -a AGE, --cache-file-max-age-seconds AGE +.. option:: -a REVAL_STATUS, --set-reval-status REVAL_STATUS - Sets the maximum age - in seconds - a cached response can be in order to be considered "fresh" - older files will be re-generated and cached. Default: 60 + Sets the ``reval_pending`` property of the server in Traffic Ops. Must be 'true' or 'false'. Requires :option:`--set-queue-status` also be set. This disables normal output. .. option:: -e ERROR_LOCATION, --log-location-error ERROR_LOCATION The file location to which to log errors. Respects the special string constants of :atc-godoc:`lib/go-log`. Default: 'stderr' -.. option:: -g, --print-generated-files +.. option:: -d DATA, --get-data DATA - If given, the names of files generated (and not proxied to Traffic Ops) will be printed to stdout, then :program:`atstccfg` will exit. + Specifies non-configuration-file data to retrieve from Traffic Ops. This disables normal output. Valid values are: -.. option:: -i INFO_LOCATION, --log-location-info INFO_LOCATION + chkconfig + Retrieves information about the services which should be running on the :term:`cache server`. The output will be in JSON-encoded format as an array of objects with these fields: - The file location to which to log information messages. Respects the special string constants of :atc-godoc:`lib/go-log`. Default: 'stderr' + :name: The name of the service. This should correspond to an existing systemd service unit file. + :value: A "chkconfig" line describing on which "run-levels" the services should be running. See the :manpage:`chkconfig(8)` manual pages for details on what this field means. + + packages + Retrieves information about the packages which should exist on the :term:`cache server`. The output will be in JSON-encoded format as an array of + objects with these fields: + + :name: The name of the package. This should hopefully be a meaningful package name for the :term:`cache server`'s package management system. + :version: The version of the package which should be installed. This might also be an empty string which means "any version will do". + + statuses + Retrieves all statuses from Traffic Ops. This is defined to be exactly the ``response`` object from the response to a GET request made to the :ref:`to-api-statuses` Traffic Ops API endpoint. + system-info + Retrieves generic information about the Traffic Control system from the :ref:`to-api-system-info` API endpoint. The output is the ``parameters`` object of the responses from GET requests to that endpoint (still JSON-encoded). + update-status + Retrieves information about the current update status using :ref:`to-api-servers-hostname-update_status`. The response is in the same format as the responses for that endpoint's GET method handler - except that that endpoint returns an array and this :program:`atstccfg` call signature returns a single one of those elements. Which one is chosen is arbitrary (hence undefined behavior when more than one server with the same hostname exists). .. option:: -h, --help Print usage information and exit. +.. option:: -i INFO_LOCATION, --log-location-info INFO_LOCATION + + The file location to which to log information messages. Respects the special string constants of :atc-godoc:`lib/go-log`. Default: 'stderr' + .. option:: -l, --list-plugins - List the loaded plugins and then exit. + List the loaded plug-ins and then exit. + +.. option:: -n NAME, --cache-host-name NAME + + Required. Specifies the (short) hostname of the :term:`cache server` for which output will be generated. Must be the server hostname in Traffic Ops, not a URL, or :abbr:`FQDN (Fully Qualified Domain Name)`. Behavior when more than one server exists with the passed hostname is undefined. -.. option:: -n, --no-cache +.. option:: -p, --traffic-ops-disable-proxy - If given, existing cache files will not be used. Cache files will still be created, existing ones just won't be used. + Bypass the Traffic Ops caching proxy and make requests directly to Traffic Ops. Has no effect if no such proxy exists. .. option:: -P TO_PASSWORD, --traffic-ops-password TO_PASSWORD - Authenticate using this password - if not given, atstccfg will attempt to use the value of the :envvar:`TO_PASS` environment variable. + Authenticate using this password - if not given, :program:`atstccfg` will attempt to use the value of the :envvar:`TO_PASSWORD` environment variable. + +.. option:: -q QUEUE_STATUS, --set-queue-status QUEUE_STATUS + + Sets the ``upd_pending`` property of the server identified by :option:`--cache-host-name` to the specified value, which must be 'true' or 'false'. Requires :option:`--set-reval-status` to also be set. .. option:: -r N, --num-retries N @@ -76,7 +111,7 @@ Options .. option:: -t TIMEOUT, --traffic-ops-timeout-milliseconds TIMEOUT - Sets the timeout - in milliseconds - for requests made to Traffic Ops. Default: 10000 + Sets the timeout - in milliseconds - for requests made to Traffic Ops. Default: 30000 .. option:: -u TO_URL, --traffic-ops-url TO_URL @@ -94,6 +129,10 @@ Options The file location to which to log warnings. Respects the special string constants of :atc-godoc:`lib/go-log`. Default: 'stderr' +.. option:: -y, --revalidate-only + + When given, :program:`atstccfg` will only emit files relevant for updating content invalidation jobs. for Apache Traffic Server implementations, this limits the output to be only files named ``regex_revalidate.config``. Has no effect if :option:`--get-data` or :option:`--set-queue-status`/:option:`--set-reval-status` is/are used. + Environment Variables --------------------- @@ -101,7 +140,7 @@ Environment Variables Defines the user as whom to authenticate with Traffic Ops. This is only used if :option:`-U`/:option:`--traffic-ops-user` is not given. -.. envvar:: TO_PASS +.. envvar:: TO_PASSWORD Defines the password to use when authenticating with Traffic Ops. This is only used if :option:`-P`/:option:`--traffic-ops-password` is not given. From ec387937762d4cc5816662ae8d556be56bf2e585 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:35:30 -0600 Subject: [PATCH 40/49] Each command on its own line --- infrastructure/cdn-in-a-box/cache/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index 8216599683..6199a47c53 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -53,7 +53,12 @@ WORKDIR /opt ADD infrastructure/cdn-in-a-box/ort /opt/ort/ ADD traffic_control/clients/python /opt/Apache-TrafficControl/ -RUN touch /var/log/ort.log && pip3 install ./Apache-TrafficControl && pip3 install ./ort && cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && mkdir -p /opt/init.d && cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort +RUN touch /var/log/ort.log && \ + pip3 install ./Apache-TrafficControl && \ + pip3 install ./ort && \ + cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && \ + mkdir -p /opt/init.d && \ + cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / ADD traffic_ops/ort/atstccfg/atstccfg /bin/ From a01ce49aa35e89983f8a38aa7e09544757c509d5 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:38:17 -0600 Subject: [PATCH 41/49] Remove now unused dependency; switch docs to mention new tests --- infrastructure/cdn-in-a-box/ort/README.rst | 2 +- infrastructure/cdn-in-a-box/ort/setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/README.rst b/infrastructure/cdn-in-a-box/ort/README.rst index bf9d9b4ca5..edc18a23b2 100644 --- a/infrastructure/cdn-in-a-box/ort/README.rst +++ b/infrastructure/cdn-in-a-box/ort/README.rst @@ -29,4 +29,4 @@ Running Tests To run tests: - python3 -m unittest discover -s tests + ./doctest-runner.py diff --git a/infrastructure/cdn-in-a-box/ort/setup.py b/infrastructure/cdn-in-a-box/ort/setup.py index 28e00c9b30..84569fc61b 100755 --- a/infrastructure/cdn-in-a-box/ort/setup.py +++ b/infrastructure/cdn-in-a-box/ort/setup.py @@ -61,7 +61,6 @@ keywords='network connection configuration TrafficControl', packages=find_packages(exclude=['contrib', 'docs', 'tests']), install_requires=['setuptools', 'typing', 'requests', 'urllib3', 'distro', 'psutil', 'Apache-TrafficControl'], - extras_require={'dev': ['unittest']}, # data_files=[('etc/crontab', ['traffic_ops_ort.crontab'])], entry_points={ 'console_scripts': [ From c31ebc18798bccacd0ab285439419a1293e5aa55 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:39:11 -0600 Subject: [PATCH 42/49] Remove extra line --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py index d0da7c3ee2..8358178787 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py @@ -103,7 +103,6 @@ def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/", *unused_ar except (KeyError, TypeError, IndexError) as e: raise ValueError from e - def __repr__(self) -> str: """ Implements ``repr(self)`` From 3498bae36a6f0d81f082151663dc41c4a81ef542 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:41:30 -0600 Subject: [PATCH 43/49] Fixed bad class reference --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 4bfdd67a54..4415e6cee7 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -160,7 +160,7 @@ def getMyPackages(self) -> typing.List[packaging.Package]: if proc.stderr.decode(): logging.error("proc.stderr.decode()") if proc.returncode == 0: - return [Package(p) for p in json.loads(proc.stdout.decode())] + return [packaging.Package(p) for p in json.loads(proc.stdout.decode())] except (ValueError, IndexError, json.JSONDecodeError, OSError, subprocess.SubprocessError) as e: logging.debug("package fetch failure: %r", e, stack_info=True, exc_info=True) From f271ac1f275dcee541fae712ee500dff67315136 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:42:07 -0600 Subject: [PATCH 44/49] Removed now-unused code --- .../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py index 4415e6cee7..86c875ae14 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py @@ -195,17 +195,6 @@ def getMyConfigFiles(self, conf:Configuration) -> typing.List[ConfigFile]: raise ConnectionError("Failed to fetch configuration files from Traffic Ops") - # Server-info sanitization is now done by atstccfg - # try: - # conf.serverInfo = ServerInfo(myFiles.info) - # # if there's a reverse proxy, switch urls. - # if conf.serverInfo.toRevProxyUrl and not conf.rev_proxy_disable: - # self._server_url = conf.serverInfo.toRevProxyUrl - # self._api_base_url = urljoin(self._server_url, '/api/%s' % self.VERSION).rstrip('/') + '/' - # return myFiles.configFiles - # except (KeyError, AttributeError, ValueError) as e: - # raise ConnectionError("Malformed response from Traffic Ops to update status request!") from e - def updateTrafficOps(self, mode:Configuration.Modes): """ Updates Traffic Ops's knowledge of this server's update status. From e8ba0eed2164b708c48d6ead509a89ba3993c1c3 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Fri, 1 May 2020 09:42:42 -0600 Subject: [PATCH 45/49] Added a space to a comment. --- infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py index 2243c3e479..95128ff73c 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/utils.py @@ -159,7 +159,7 @@ def parse_multipart(raw: str) -> typing.List[typing.Tuple[str, str]]: if not boundary: raise ValueError("'Content-Type' header missing 'boundary' parameter") - chunks = raw.split(f"--{boundary}")[1:] #ignore prologue + chunks = raw.split(f"--{boundary}")[1:] # ignore prologue if chunks[-1].strip() != "--": logging.warning("Final chunk appears invalid - possible bad message payload") else: From 1d98fb3a147ad19773303bdab3062f47d92df93b Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 16 Jun 2020 14:03:50 -0600 Subject: [PATCH 46/49] Changed cache builds to use ORT RPM --- infrastructure/cdn-in-a-box/Makefile | 12 +++++++----- infrastructure/cdn-in-a-box/cache/Dockerfile | 10 +++++++++- infrastructure/cdn-in-a-box/edge/run.sh | 2 +- infrastructure/cdn-in-a-box/mid/run.sh | 2 +- .../cdn-in-a-box/ort/traffic_ops_ort.crontab | 2 +- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index 8bb1e8645c..dde9a26b62 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -42,8 +42,7 @@ SPECIAL_SEASONING := $(TOMCAT_VERSION).$(TOMCAT_RELEASE)-$(BUILD_NUMBER).$(RHEL_ TO_SOURCE := $(wildcard ../../traffic_ops/**/*) TO_SOURCE += $(wildcard ../../traffic_ops_db/**/*) -ORT_SOURCE:= $(wildcard ../../traffic_ops/ort/atstccfg/**/*.go) -ORT_SOURCE+= $(wildcard ../../traffic_ops/client/*.go) +ORT_SOURCE:= $(wildcard ../../traffic_ops_ort/atstccfg/**/*.go) ORT_SOURCE+= $(wildcard ../../lib/**/*.go) TM_SOURCE := $(wildcard ../../traffic_monitor/**/*) TP_SOURCE := $(wildcard ../../traffic_portal/**/*) @@ -75,8 +74,8 @@ traffic_router/tomcat.rpm: ../../dist/tomcat-$(SPECIAL_SEASONING) cp -f $? $@ traffic_stats/traffic_stats.rpm: ../../dist/traffic_stats-$(SPECIAL_SAUCE) cp -f $? $@ -../../traffic_ops/ort/atstccfg/atstccfg: $(ORT_SOURCE) - go build -o $@ ../../traffic_ops/ort/atstccfg +cache/traffic_ops_ort.rpm: ../../dist/traffic_ops_ort-$(SPECIAL_SAUCE) + cp -f $? $@ # Dist rpms ../../dist/traffic_monitor-$(SPECIAL_SAUCE): $(TM_SOURCE) @@ -94,8 +93,11 @@ traffic_stats/traffic_stats.rpm: ../../dist/traffic_stats-$(SPECIAL_SAUCE) ../../dist/traffic_stats-$(SPECIAL_SAUCE): $(TS_SOURCE) ../../pkg $(PKG_FLAGS) traffic_stats_build +../../dist/traffic_ops_ort-$(SPECIAL_SAUCE): $(ORT_SOURCE) + ../../pkg $(PKG_FLAGS) traffic_ops_ort_build + clean: - $(RM) traffic_monitor/traffic_monitor.rpm traffic_ops/traffic_ops.rpm traffic_portal/traffic_portal.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm edge/traffic_ops_ort.rpm mid/traffic_ops_ort.rpm traffic_stats/traffic_stats.rpm + $(RM) traffic_monitor/traffic_monitor.rpm traffic_ops/traffic_ops.rpm traffic_portal/traffic_portal.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm cache/traffic_ops_ort.rpm traffic_stats/traffic_stats.rpm very-clean: clean $(warning This will destroy ALL OUTPUT RPMS IN 'dist'. Please be sure this is what you want) diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index 6199a47c53..8cc64d4a4e 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -25,6 +25,7 @@ FROM centos:7 EXPOSE 80 + ADD https://ci.trafficserver.apache.org/RPMS/CentOS7/trafficserver-7.1.4-2.el7.x86_64.rpm /trafficserver.rpm ADD https://ci.trafficserver.apache.org/RPMS/CentOS7/trafficserver-devel-7.1.4-2.el7.x86_64.rpm /trafficserver-devel.rpm @@ -32,6 +33,7 @@ RUN yum install -y bind-utils kyotocabinet-libs epel-release initscripts iproute RUN yum install -y /trafficserver.rpm /trafficserver-devel.rpm jq python36-psutil python36-typing python36-setuptools python36-pip logrotate && yum clean all RUN python3 -m pip install --upgrade pip && python3 -m pip install requests urllib3 distro + ADD traffic_server/plugins/astats_over_http/astats_over_http.c traffic_server/plugins/astats_over_http/Makefile.am / RUN tsxs -v -c astats_over_http.c -o astats_over_http.so @@ -61,5 +63,11 @@ RUN touch /var/log/ort.log && \ cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / -ADD traffic_ops/ort/atstccfg/atstccfg /bin/ + +ARG ORT_RPM=infrastructure/cdn-in-a-box/cache/traffic_ops_ort.rpm +ADD $ORT_RPM / +RUN yum install -y /$(basename $ORT_RPM) &&\ + rm /$(basename $ORT_RPM) && \ + yum -y clean all + CMD /run.sh diff --git a/infrastructure/cdn-in-a-box/edge/run.sh b/infrastructure/cdn-in-a-box/edge/run.sh index 80701d350d..449ee8cdfe 100755 --- a/infrastructure/cdn-in-a-box/edge/run.sh +++ b/infrastructure/cdn-in-a-box/edge/run.sh @@ -75,7 +75,7 @@ until [[ $(to-get api/2.0/cdns/name/$CDN_NAME/sslkeys | jq '.response | length') done # Leaves the container hanging open in the event of a failure for debugging purposes -traffic_ops_ort -kl ALL BADASS || { echo "Failed"; } +PATH="$PATH:/opt/ort" traffic_ops_ort -kl ALL BADASS || { echo "Failed"; } envsubst < "/etc/cron.d/traffic_ops_ort-cron-template" > "/var/spool/cron/root" && rm -f "/etc/cron.d/traffic_ops_ort-cron-template" crontab "/var/spool/cron/root" diff --git a/infrastructure/cdn-in-a-box/mid/run.sh b/infrastructure/cdn-in-a-box/mid/run.sh index fb9ab49c09..e339d974e6 100755 --- a/infrastructure/cdn-in-a-box/mid/run.sh +++ b/infrastructure/cdn-in-a-box/mid/run.sh @@ -69,7 +69,7 @@ while [[ -z "$(testenrolled)" ]]; do done # Leaves the container hanging open in the event of a failure for debugging purposes -traffic_ops_ort -k BADASS ALL "https://$TO_FQDN:$TO_PORT" "$TO_ADMIN_USER:$TO_ADMIN_PASSWORD" || { echo "Failed"; } +PATH="$PATH:/opt/ort" traffic_ops_ort -k BADASS ALL "https://$TO_FQDN:$TO_PORT" "$TO_ADMIN_USER:$TO_ADMIN_PASSWORD" || { echo "Failed"; } envsubst < "/etc/cron.d/traffic_ops_ort-cron-template" > "/etc/cron.d/traffic_ops_ort-cron" && rm -f "/etc/cron.d/traffic_ops_ort-cron-template" chmod "0644" "/etc/cron.d/traffic_ops_ort-cron" && crontab "/etc/cron.d/traffic_ops_ort-cron" diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab index 5ec6890207..d980655b66 100644 --- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab +++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab @@ -14,4 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -*/1 * * * * /usr/local/bin/traffic_ops_ort -k --dispersion 0 SYNCDS ALL $TO_URL $TO_USER:$TO_PASSWORD >> /var/log/ort.log 2>&1 +*/1 * * * * PATH=$PATH:/opt/ort /usr/local/bin/traffic_ops_ort -k --dispersion 0 SYNCDS ALL $TO_URL $TO_USER:$TO_PASSWORD >> /var/log/ort.log 2>&1 From 44c412d011c1ace1e4a1a3e22afdc810e359572b Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 16 Jun 2020 14:34:46 -0600 Subject: [PATCH 47/49] Added comment about which TO_PASS* env var is preferred --- traffic_ops_ort/atstccfg/config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/traffic_ops_ort/atstccfg/config/config.go b/traffic_ops_ort/atstccfg/config/config.go index 1b107d1b42..a725095ccf 100644 --- a/traffic_ops_ort/atstccfg/config/config.go +++ b/traffic_ops_ort/atstccfg/config/config.go @@ -137,6 +137,10 @@ func GetCfg() (Cfg, error) { if toUser == "" { toUser = os.Getenv("TO_USER") } + + // TO_PASSWORD is preferred over TO_PASS, as it's the one commonly used by + // Traffic Control tools. Hopefully, we'll be able to get rid of TO_PASS + // entirely in the near future, to make this less confusing. if toPass == "" { toPass = os.Getenv("TO_PASS") } From 8b823a0a589865f07e62ae24bef6ca19e250d80a Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Tue, 16 Jun 2020 18:24:14 -0600 Subject: [PATCH 48/49] Fixed erroneous 'all' target --- infrastructure/cdn-in-a-box/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile index dde9a26b62..c3e2acc28a 100644 --- a/infrastructure/cdn-in-a-box/Makefile +++ b/infrastructure/cdn-in-a-box/Makefile @@ -52,7 +52,7 @@ TS_SOURCE := $(wildcard ../../traffic_stats/**/*) .PHONY: clean very-clean all nearly-all debug # Default target; builds all pre-requisite rpms from source trees -all: ../../traffic_ops/ort/atstccfg/atstccfg traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm +all: cache/traffic_ops_ort.rpm traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm debug: PKG_FLAGS += -d ifneq ($(MAKECMDGOALS), debug) From 0b08b8a6c388044f5baaa7c4277dc91f7690af38 Mon Sep 17 00:00:00 2001 From: ocket8888 Date: Wed, 24 Jun 2020 11:15:10 -0600 Subject: [PATCH 49/49] Added a shim for bad atstccfg ssl behavior --- infrastructure/cdn-in-a-box/cache/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infrastructure/cdn-in-a-box/cache/Dockerfile b/infrastructure/cdn-in-a-box/cache/Dockerfile index 8cc64d4a4e..eac568dbb4 100644 --- a/infrastructure/cdn-in-a-box/cache/Dockerfile +++ b/infrastructure/cdn-in-a-box/cache/Dockerfile @@ -37,7 +37,9 @@ RUN python3 -m pip install --upgrade pip && python3 -m pip install requests urll ADD traffic_server/plugins/astats_over_http/astats_over_http.c traffic_server/plugins/astats_over_http/Makefile.am / RUN tsxs -v -c astats_over_http.c -o astats_over_http.so -RUN mkdir -p /usr/libexec/trafficserver /opt/ort && tsxs -v -o astats_over_http.so -i + +# The symbolic link here is a shim for broken atstccfg behavior - remove when it's fixed. +RUN mkdir -p /usr/libexec/trafficserver /opt/ort /opt/trafficserver/etc/trafficserver/ /opt/init.d && ln -s /opt/trafficserver/etc/trafficserver/ssl /etc/trafficserver/ssl && tsxs -v -o astats_over_http.so -i RUN yum remove -y gcc-c++ glibc-devel autoconf automake libtool && rm -f /astats_over_http.c /Makefile.am @@ -59,7 +61,6 @@ RUN touch /var/log/ort.log && \ pip3 install ./Apache-TrafficControl && \ pip3 install ./ort && \ cp ort/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template && \ - mkdir -p /opt/init.d && \ cp ort/traffic_ops_ort.logrotate /etc/logrotate.d/ort ADD infrastructure/cdn-in-a-box/cache/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json /