From 385e4bd5d60f80167abaf77b6730ccf7c1b5f839 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:03:40 +0200 Subject: [PATCH 01/53] fixed containers not being global --- app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app.py b/app.py index 444ab34..03b356e 100644 --- a/app.py +++ b/app.py @@ -66,6 +66,7 @@ def updateEnabledContainers(): "added": [] } + global containers newContainers = [] try: From 15385a97a75debca757aeb829c273f09db397586 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:08:16 +0200 Subject: [PATCH 02/53] fixed typo --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 03b356e..db97867 100644 --- a/app.py +++ b/app.py @@ -46,8 +46,8 @@ def getDiff(old, new): added = new_set - old_set return { - removed: list(removed), - added: list(added) + "removed": list(removed), + "added": list(added) } def getHostsFromLabels(labels): From 2b99b2d7c08369a371a9f2a44cd6bf1b6896c82d Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:11:13 +0200 Subject: [PATCH 03/53] fixed misunderstanding of count() --- app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index db97867..f4426c7 100644 --- a/app.py +++ b/app.py @@ -76,10 +76,10 @@ def updateEnabledContainers(): containerDiff = getDiff(containers, newContainers) - logger.info(f"Found {newContainers.count()} Containers") + logger.info(f"Found {newContainers.__len__()} Containers") - logger.debug(f"Found {containerDiff.get("added").count()} added Containers") - logger.debug(f"Found {containerDiff.get("removed").count()} removed Containers") + logger.debug(f"Found {containerDiff.get("added").__len__()} added Containers") + logger.debug(f"Found {containerDiff.get("removed").__len__()} removed Containers") # Threading # with containerToHostLock and containerLock: @@ -174,7 +174,7 @@ def main(): globalDiff = cleanDiff(globalDiff) # Check if there is actually any Diff - if globalDiff.get("removed").count() + globalDiff.get("added").count() <= 0: + if globalDiff.get("removed").__len__() + globalDiff.get("added").__len__() <= 0: logger.debug("No Changes were made, skipping...") return From ecf05d529fdb36001e0206a091272c4f6567d1c9 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:17:13 +0200 Subject: [PATCH 04/53] fixed unset variables --- app.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index f4426c7..7a3fbee 100644 --- a/app.py +++ b/app.py @@ -43,7 +43,7 @@ def getDiff(old, new): old_set, new_set = set(old), set(new) removed = old_set - new_set - added = new_set - old_set + added = new_set - old_set return { "removed": list(removed), @@ -123,14 +123,16 @@ def updateEnabledContainers(): return diffs def cleanDiff(diff): - both = diff.get("removed") and diff.get("added") + removed, added = set(diff.get("removed")), set(diff.get("added")) + + both = removed & added removed -= both added -= both return { - removed: list(removed), - added: list(added) + "removed": list(removed), + "added": list(added) } def exitContainer(): From 04d574f4293968989b2f981beea44342e09640b8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:28:22 +0200 Subject: [PATCH 05/53] fixed loop exiting to soon --- app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app.py b/app.py index 7a3fbee..5953025 100644 --- a/app.py +++ b/app.py @@ -178,8 +178,7 @@ def main(): # Check if there is actually any Diff if globalDiff.get("removed").__len__() + globalDiff.get("added").__len__() <= 0: logger.debug("No Changes were made, skipping...") - - return + continue logger.info(f"Sending Diff to {ENDPOINT} with{"out" if not ENDPOINT_KEY else ""} Auth") From 01865b47628942c36e6eed4c4a45c408567eef6f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 18:40:19 +0200 Subject: [PATCH 06/53] debugging --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 5953025..237e44f 100644 --- a/app.py +++ b/app.py @@ -184,8 +184,8 @@ def main(): response = sendDiffToEndpoint(globalDiff) - if not response.ok: - logger.error(f"Endpoint responded with {response.status_code} NOT OK") + if response: + logger.error(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") if __name__ == '__main__': logger.setLevel(level=LOG_LEVEL) From 69bf9bc00b112f166d5b66954a65a3f111768a58 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 18:45:55 +0200 Subject: [PATCH 07/53] fix newly added containers not contributing to global diff --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index 237e44f..17cbd30 100644 --- a/app.py +++ b/app.py @@ -105,6 +105,8 @@ def updateEnabledContainers(): containersToHosts[container.id] = hosts else: logger.debug(f"Added {container.name}") + + diffs.get("added").extend(hosts) containersToHosts[container.id] = hosts From 9476e64da4f34ad764e79d872f1c1a0c4d08fff4 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 18:48:16 +0200 Subject: [PATCH 08/53] further debug --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 17cbd30..ada922a 100644 --- a/app.py +++ b/app.py @@ -187,7 +187,9 @@ def main(): response = sendDiffToEndpoint(globalDiff) if response: - logger.error(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") + logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") + else: + logger.error(f"Endpoint did not respond") if __name__ == '__main__': logger.setLevel(level=LOG_LEVEL) From cd14c5e2a984a3b9f02f42eeaa0791049150b7ff Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 19:05:16 +0200 Subject: [PATCH 09/53] revert --- app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.py b/app.py index ada922a..44420bd 100644 --- a/app.py +++ b/app.py @@ -186,10 +186,7 @@ def main(): response = sendDiffToEndpoint(globalDiff) - if response: - logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") - else: - logger.error(f"Endpoint did not respond") + logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") if __name__ == '__main__': logger.setLevel(level=LOG_LEVEL) From a93f67a9163a60cb820b251b298397ab1d73c44c Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 23 Aug 2025 19:08:25 +0200 Subject: [PATCH 10/53] fixed wrong json key --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 44420bd..9b1382f 100644 --- a/app.py +++ b/app.py @@ -143,7 +143,7 @@ def exitContainer(): os.kill(os.getpid(), signal.SIGTERM) def sendDiffToEndpoint(diff): - data = { "server": SERVER_NAME, "diff": diff } + data = { "serverName": SERVER_NAME, "diff": diff } headers = {} From 2511caa35980fb750b2083c7a0760d436867eb94 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:19:22 +0200 Subject: [PATCH 11/53] added Full Discoveries to give the endpoint a complete Update --- .github/templates/README.template.md | 17 ++++- .gitignore | 3 +- app.py | 95 ++++++++++++++++++++++++---- docker-compose.yaml | 1 + 4 files changed, 100 insertions(+), 16 deletions(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 0e1be58..15ba34b 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -56,7 +56,22 @@ If no Key is provided ServDiscovery will leave out the Authorization Header. ### DISCOVERY_INTERVAL -The Discovery Interval sets the Interval of which ServDiscovery will update the Endpoint, etc. +The Discovery Interval sets the Interval (in seconds) of which ServDiscovery will update the provided Endpoint. + +Default: `60` + +### FULL_DISCOVERY_INTERVAL + +Sets the Interval for Full Disoveries. +This tells ServDiscovery to send a Full Update of all the activate Containers in the `added` JSON Key. + +> [!IMPORTANT] +> Must be set to a **fraction of** DISCOVERY_INTERVAL. +> (example: `DISCOVERY_INTERVAL * 2`) +> `FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL` must result in an int. +> `FULL_DISCOVERY_INTERVAL` must be bigger than `DISCOVERY_INTERVAL`. + +Default: **Disabled** ## Contributing diff --git a/.gitignore b/.gitignore index c66ba3a..aebd04d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env -.venv \ No newline at end of file +.venv +*.lua \ No newline at end of file diff --git a/app.py b/app.py index 9b1382f..0cb1c89 100644 --- a/app.py +++ b/app.py @@ -29,6 +29,9 @@ ENDPOINT_KEY = os.getenv("ENDPOINT_KEY") DISCOVERY_INTERVAL = os.getenv("DISCOVERY_INTERVAL") +FULL_DISCOVERY_INTERVAL = os.getenv("FULL_DISCOVERY_INTERVAL") + +fullDiscoveryRatio = 0 dockerClient = DockerClient(base_url="unix://var/run/docker.sock") @@ -60,19 +63,25 @@ def getHostsFromLabels(labels): return hosts -def updateEnabledContainers(): +def getEnabledContainers(): + newContainers = [] + + try: + newContainers = dockerClient.containers.list(filters={"label": "discovery.enable=true"}) + except Exception as e: + logger.error(f"Error fetching containers: {str(e)}") + + return newContainers + +def getDiscovery(): diffs = { "removed": [], "added": [] } global containers - newContainers = [] - try: - newContainers = dockerClient.containers.list(filters={"label": "discovery.enable=true"}) - except Exception as e: - logger.error(f"Error fetching containers: {str(e)}") + newContainers = getEnabledContainers() containerDiff = getDiff(containers, newContainers) @@ -124,6 +133,23 @@ def updateEnabledContainers(): return diffs +def getFullDiscovery(): + diffs = { + "removed": [], + "added": [] + } + + newContainers = getEnabledContainers() + + logger.info(f"Found {newContainers.__len__()} Containers") + + for container in newContainers: + hosts = getHostsFromLabels(container.labels) + + diffs.get("added").extend(hosts) + + return diffs + def cleanDiff(diff): removed, added = set(diff.get("removed")), set(diff.get("added")) @@ -163,21 +189,49 @@ def sendDiffToEndpoint(diff): # thread = threading.Thread(target=main, daemon=True) # thread.start() +def doDiscovery(): + logger.debug("Starting Discovery") + + globalDiff = getDiscovery() + + logger.debug("Cleaning Diff") + + return cleanDiff(globalDiff) + +def doFullDiscovery(): + logger.debug("Starting Full Discovery") + + globalDiff = getFullDiscovery() + + return globalDiff + def main(): + i = 0 while True: - logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...") + isFullDiscovery = False - sleep(DISCOVERY_INTERVAL) - - logger.info("Starting Discovery") + if i == fullDiscoveryRatio: + isFullDiscovery = True + i = 0 - globalDiff = updateEnabledContainers() + if not isFullDiscovery: + logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...") + else: + logger.info(f"Starting Full Discover in {DISCOVERY_INTERVAL}...") - logger.debug("Cleaning Diff") + sleep(DISCOVERY_INTERVAL) - globalDiff = cleanDiff(globalDiff) + globalDiff = { + "removed": [], + "added": [] + } - # Check if there is actually any Diff + if not isFullDiscovery: + globalDiff = doDiscovery() + else: + globalDiff = doFullDiscovery() + + # Check if there are any Changes if globalDiff.get("removed").__len__() + globalDiff.get("added").__len__() <= 0: logger.debug("No Changes were made, skipping...") continue @@ -188,6 +242,8 @@ def main(): logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") + i += 1 + if __name__ == '__main__': logger.setLevel(level=LOG_LEVEL) @@ -203,6 +259,17 @@ def main(): logger.warning(f"No DISCOVERY_INTERVAL set, using 30sec as default") DISCOVERY_INTERVAL = 30 + if not FULL_DISCOVERY_INTERVAL: + logger.warning(f"No FULL_DISCOVERY_INTERVAL set, disregarding Full Discoveries") + FULL_DISCOVERY_INTERVAL = 0 + elif FULL_DISCOVERY_INTERVAL % DISCOVERY_INTERVAL != 0: + logger.warning(f"FULL_DISCOVERY_INTERVAL is not a valid Integer, rounding to nearest Int") + FULL_DISCOVERY_INTERVAL = round(FULL_DISCOVERY_INTERVAL) + + if FULL_DISCOVERY_INTERVAL != 0: + fullDiscoveryRatio = FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL + logger.debug(f"FULL_DISCOVERY_INTERVAL ratio set to {fullDiscoveryRatio}") + if not ENDPOINT_KEY or ENDPOINT_KEY == "": logger.warning(f"No ENDPOINT_KEY set, requests may be denied") diff --git a/docker-compose.yaml b/docker-compose.yaml index a7112bb..ad61f32 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,6 +5,7 @@ services: environment: ENDPOINT: https://mydomain.com/ENDPOINT ENDPOINT_KEY: MY_VERY_SECURE_KEY + ALIVE_UPDATE_INTERVAL: 60 SERVER_NAME: server-1 volumes: - /var/run/docker.sock:/var/run/docker.sock From 0eaa24321ffd9c6a35a9e3f4f69fd3e24e346b6e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 30 Aug 2025 09:19:34 +0000 Subject: [PATCH 12/53] Update README.md --- README.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e5fffb4..081afb7 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ services: environment: ENDPOINT: https://mydomain.com/ENDPOINT ENDPOINT_KEY: MY_VERY_SECURE_KEY + ALIVE_UPDATE_INTERVAL: 60 SERVER_NAME: server-1 volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -99,7 +100,22 @@ If no Key is provided ServDiscovery will leave out the Authorization Header. ### DISCOVERY_INTERVAL -The Discovery Interval sets the Interval of which ServDiscovery will update the Endpoint, etc. +The Discovery Interval sets the Interval (in seconds) of which ServDiscovery will update the provided Endpoint. + +Default: `60` + +### FULL_DISCOVERY_INTERVAL + +Sets the Interval for Full Disoveries. +This tells ServDiscovery to send a Full Update of all the activate Containers in the `added` JSON Key. + +> [!IMPORTANT] +> Must be set to a **fraction of** DISCOVERY_INTERVAL. +> (example: `DISCOVERY_INTERVAL * 2`) +> `FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL` must result in an int. +> `FULL_DISCOVERY_INTERVAL` must be bigger than `DISCOVERY_INTERVAL`. + +Default: **Disabled** ## Contributing From 82700f34ff769e3307dcbb1fd9a31ac111b26971 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:25:23 +0200 Subject: [PATCH 13/53] fix typo in README --- .github/templates/README.template.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 15ba34b..95a2547 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -36,7 +36,7 @@ ServDiscovery sends requests to the Endpoint as a **JSON HTTP Request**: { { file.examples/payload.json } } ``` -This example tell the Endpoint that... +This example tells the Endpoint that... | Available | Unavailable | | -------------------- | --------------------------- | @@ -44,6 +44,8 @@ This example tell the Endpoint that... | website.mydomain.com | website-backup.mydomain.com | | auth.mydomain.com | auth-backup.mydomain.com | +... is (un)available + This way (if the Endpoint is used by a LoadBalancer) the Owner of the Endpoint can now delete the `*-backup.mydomain.com` records from a Registry, thus updating the list of routable Containers / Services. From 81f8e487c2b0b9b4d8f62d29a9dece8de64a7382 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 30 Aug 2025 09:25:33 +0000 Subject: [PATCH 14/53] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 081afb7..8225881 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ ServDiscovery sends requests to the Endpoint as a **JSON HTTP Request**: } ``` -This example tell the Endpoint that... +This example tells the Endpoint that... | Available | Unavailable | | -------------------- | --------------------------- | @@ -88,6 +88,8 @@ This example tell the Endpoint that... | website.mydomain.com | website-backup.mydomain.com | | auth.mydomain.com | auth-backup.mydomain.com | +... is (un)available + This way (if the Endpoint is used by a LoadBalancer) the Owner of the Endpoint can now delete the `*-backup.mydomain.com` records from a Registry, thus updating the list of routable Containers / Services. From 575c3b435f470a4b7dab20124ab5c9587d3f3280 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:51:17 +0200 Subject: [PATCH 15/53] convert to int --- app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 0cb1c89..fe5d782 100644 --- a/app.py +++ b/app.py @@ -258,11 +258,16 @@ def main(): if not DISCOVERY_INTERVAL: logger.warning(f"No DISCOVERY_INTERVAL set, using 30sec as default") DISCOVERY_INTERVAL = 30 + else: + DISCOVERY_INTERVAL = int(DISCOVERY_INTERVAL) if not FULL_DISCOVERY_INTERVAL: logger.warning(f"No FULL_DISCOVERY_INTERVAL set, disregarding Full Discoveries") FULL_DISCOVERY_INTERVAL = 0 - elif FULL_DISCOVERY_INTERVAL % DISCOVERY_INTERVAL != 0: + else: + FULL_DISCOVERY_INTERVAL = int(FULL_DISCOVERY_INTERVAL) + + if FULL_DISCOVERY_INTERVAL % DISCOVERY_INTERVAL != 0: logger.warning(f"FULL_DISCOVERY_INTERVAL is not a valid Integer, rounding to nearest Int") FULL_DISCOVERY_INTERVAL = round(FULL_DISCOVERY_INTERVAL) From 23eb46bff8ff42287acf0955a013f2f1fa428a16 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 30 Aug 2025 11:55:18 +0200 Subject: [PATCH 16/53] round to int --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index fe5d782..01975e9 100644 --- a/app.py +++ b/app.py @@ -272,7 +272,7 @@ def main(): FULL_DISCOVERY_INTERVAL = round(FULL_DISCOVERY_INTERVAL) if FULL_DISCOVERY_INTERVAL != 0: - fullDiscoveryRatio = FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL + fullDiscoveryRatio = round(FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL) logger.debug(f"FULL_DISCOVERY_INTERVAL ratio set to {fullDiscoveryRatio}") if not ENDPOINT_KEY or ENDPOINT_KEY == "": From 070f1b01321eefaf5718608137b3b53aab6e6584 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Sat, 30 Aug 2025 12:03:18 +0200 Subject: [PATCH 17/53] moved i to the top of the func --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 01975e9..8fd8fa7 100644 --- a/app.py +++ b/app.py @@ -214,6 +214,8 @@ def main(): isFullDiscovery = True i = 0 + i += 1 + if not isFullDiscovery: logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...") else: @@ -242,8 +244,6 @@ def main(): logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") - i += 1 - if __name__ == '__main__': logger.setLevel(level=LOG_LEVEL) From 21ac2bfc6f0e0903d3d38dcb9801ca70c4fcfc7b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Mon, 6 Oct 2025 20:21:20 +0200 Subject: [PATCH 18/53] Update app.py --- app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 8fd8fa7..1a2b417 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ import sys from time import sleep +# Threading # import threading logger = logging.getLogger("ServDiscovery") @@ -278,4 +279,5 @@ def main(): if not ENDPOINT_KEY or ENDPOINT_KEY == "": logger.warning(f"No ENDPOINT_KEY set, requests may be denied") - main() \ No newline at end of file + + main() From 9f4226906be6e8c8579f1e46829b0e95991a2c86 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:42:46 +0100 Subject: [PATCH 19/53] initial port --- .dockerignore | 17 +- .github/templates/README.template.md | 104 +++++++---- .gitignore | 36 +++- examples/haproxy/haproxy.cfg | 20 +++ examples/haproxy/haproxy.lua | 176 ++++++++++++++++++ examples/whoami.docker-compose.yaml | 3 +- go.mod | 33 ++++ go.sum | 47 +++++ internals/config/config.go | 42 +++++ internals/config/structure/structure.go | 10 ++ internals/discovery/diff.go | 87 +++++++++ internals/discovery/discovery.go | 225 ++++++++++++++++++++++++ internals/docker/client.go | 40 +++++ internals/docker/docker.go | 30 ++++ main.go | 89 ++++++++++ 15 files changed, 915 insertions(+), 44 deletions(-) create mode 100644 examples/haproxy/haproxy.cfg create mode 100644 examples/haproxy/haproxy.lua create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internals/config/config.go create mode 100644 internals/config/structure/structure.go create mode 100644 internals/discovery/diff.go create mode 100644 internals/discovery/discovery.go create mode 100644 internals/docker/client.go create mode 100644 internals/docker/docker.go create mode 100644 main.go diff --git a/.dockerignore b/.dockerignore index 6969a38..9ab66d0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,16 @@ -!app.py \ No newline at end of file +# env file +*.env + +# Exclude git folders +.git* +!.github + +# Ignore yml files +*.yaml +*.yml + +# Markdown files +*.md + +# Include data/ +!data/* \ No newline at end of file diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 95a2547..2805eeb 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -1,85 +1,113 @@ +
+ + GitHub release + + + GitHub stars + + + Docker image size + + + Docker image Pulls + + + License: MIT + +
+ # ServDiscovery -ServDiscovery is a Discovery Service that keeps an Endpoint updated with active Hosts (of Services). +**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services—perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. ## Installation -> [!NOTE] -> ServDiscovery only works with Traefik and not with **any** other Reverse Proxy due to `discover.enable` lable +> [!IMPORTANT] +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discover.enable` label requirement. -Get the latest `docker-compose.yaml` file: +Get the latest `docker-compose.yaml`: ```yaml -{ { file.docker-compose.yaml } } +{{{ #://docker-compose.yaml }}} ``` +Then spin it up: + ```bash docker compose up -d ``` +Your discovery service is now live! 🎉 + ## Usage -Take this little `whoami` Container as an Example: +Let`s take a simple `whoami` container as an example: ```yaml -{ { file.examples/whoami.docker-compose.yaml } } +{{{ #://examples/whoami.docker-compose.yaml } } ``` -Whenever a new **Host-Rule** gets added / modified ServDiscovery will update the set Endpoint to notify of any new changes. -This way the Endpoint can correctly route to different Hosts based on **SNI / Hostnames**. +Whenever a new **Host-Rule** is added or updated, ServDiscovery will **automatically notify the configured endpoint**. +This ensures the endpoint can correctly route traffic based on **SNI / Hostnames**. -## Endpoint +## Endpoint Integration -ServDiscovery sends requests to the Endpoint as a **JSON HTTP Request**: +ServDiscovery communicates with your endpoint via **JSON HTTP Requests**: ```json -{ { file.examples/payload.json } } +{{{ #://examples/payload.json }}} ``` -This example tells the Endpoint that... +Example explanation: -| Available | Unavailable | +| ✅ Available | ❌ Unavailable | | -------------------- | --------------------------- | | whoami.mydomain.com | whoami-backup.mydomain.com | | website.mydomain.com | website-backup.mydomain.com | | auth.mydomain.com | auth-backup.mydomain.com | -... is (un)available +This allows the endpoint (e.g., a load balancer) to remove `\*-backup` records from your registry and **update routable containers/services automatically**. -This way (if the Endpoint is used by a LoadBalancer) the Owner of the Endpoint can now delete the `*-backup.mydomain.com` records from a Registry, -thus updating the list of routable Containers / Services. +### Integrations -## Configuration +You can find example integrations inside of [examples/](./examples). -### ENDPOINT_KEY - -The Endpoint Key is provided in the Authorization Header (via Bearer) during the POST request between the Endpoint and ServDiscovery. -If no Key is provided ServDiscovery will leave out the Authorization Header. - -### DISCOVERY_INTERVAL +## Configuration -The Discovery Interval sets the Interval (in seconds) of which ServDiscovery will update the provided Endpoint. +### `ENDPOINT_KEY` -Default: `60` +The endpoint key is used in the `Authorization` header (Bearer token) when ServDiscovery sends POST requests. +If no key is provided, the header is omitted. -### FULL_DISCOVERY_INTERVAL +### `DISCOVERY_INTERVAL` -Sets the Interval for Full Disoveries. -This tells ServDiscovery to send a Full Update of all the activate Containers in the `added` JSON Key. +Time (in seconds) between updates to your endpoint. +**Default:** `60` seconds -> [!IMPORTANT] -> Must be set to a **fraction of** DISCOVERY_INTERVAL. -> (example: `DISCOVERY_INTERVAL * 2`) -> `FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL` must result in an int. -> `FULL_DISCOVERY_INTERVAL` must be bigger than `DISCOVERY_INTERVAL`. +### `ALIVE_INTERVAL` -Default: **Disabled** +Time (in seconds) between full alive discoveries. ServDiscovery sends a **complete update** of all active containers in the `added` JSON key. +**Default:** `120` seconds ## Contributing -Found a bug or have new ideas or enhancements for this Project? -Feel free to open up an issue or create a Pull Request! +Found a bug or have a brilliant idea? Contributions are welcome! Open an **issue** or create a **pull request** — your help makes this project better. ## License -[MIT](https://choosealicense.com/licenses/mit/) +This project is licensed under the [MIT License](./LICENSE). diff --git a/.gitignore b/.gitignore index aebd04d..69c120e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,33 @@ -.env -.venv -*.lua \ No newline at end of file +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Code coverage profiles and other test artifacts +*.out +coverage.* +*.coverprofile +profile.cov + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +*.env + +# Editor/IDE +# .idea/ +.vscode/ + +# Exclude git folders +.git* +!.github \ No newline at end of file diff --git a/examples/haproxy/haproxy.cfg b/examples/haproxy/haproxy.cfg new file mode 100644 index 0000000..69a30fb --- /dev/null +++ b/examples/haproxy/haproxy.cfg @@ -0,0 +1,20 @@ +global + lua-prepend-path /tmp/haproxy/lua/?.lua + # Load discovery script + lua-load /tmp/haproxy/lua/snidiscovery.lua + # Load json helper script + # Download here: https://github.com/rxi/json.lua/blob/master/json.lua + lua-load /tmp/haproxy/lua/json.lua + +frontend sni-discovery_frontend + bind *:4676 + mode http + option http-keep-alive + + acl is_private_range src 192.168.0.0/16 172.16.0.0/12 + acl is_authenticated hdr(Authorization) -i "Bearer ENDPOINT_KEY" + + http-request deny if !is_private_range || !is_authenticated + + # Apply discovery routing + http-request use-service lua.update_mapping diff --git a/examples/haproxy/haproxy.lua b/examples/haproxy/haproxy.lua new file mode 100644 index 0000000..c2a5e23 --- /dev/null +++ b/examples/haproxy/haproxy.lua @@ -0,0 +1,176 @@ +local json = require("json") + +-- delete Host after some time (add a few seconds to remove any race-conditioning) +local host_server_alive_time = 65 + +-- interval for stale cleanup +local host_server_stale_cleanup_interval = 5 + +-- mapping table: host.domain.com -> [ { serverName: "server1", aliveUntil: 600 }, { serverName: "server2", aliveUntil: 300 }, ... ] +local host_servers = host_servers or {} + +-- helper to update host-server mapping +local function apply_diff(serverName, diff) + -- Add hosts + if diff.added then + local currentTime = core.now() + + for _, host in ipairs(diff.added) do + host_servers[host] = host_servers[host] or {} + + local exists = false + for _, server in ipairs(host_servers[host]) do + if server.serverName == serverName then exists = true break end + end + + local serverObject = { + serverName = serverName, + aliveUntil = currentTime.sec + host_server_alive_time + } + + if not exists then + core.Info("Added Server "..serverName.." in "..host..". Reason: Diff-Added") + + table.insert(host_servers[host], serverObject) + else + -- Update Alive Time + for i = 1, #host_servers[host] do + local server = host_servers[host][i] + + if server.serverName == serverName then + core.Info("Bumped "..serverName.."'s Alive Time in "..host.." by "..host_server_alive_time..". Reason: Diff-Added + Update") + + host_servers[host][i] = serverObject + end + end + end + end + end + + -- Remove hosts + if diff.removed then + for _, host in ipairs(diff.removed) do + if host_servers[host] then + for i = #host_servers[host], 1, -1 do + local server = host_servers[host][i] + + if server.serverName == serverName then + core.Info("Removed Server "..serverName.." from "..host..". Reason: Diff-Removed") + + table.remove(host_servers[host], i) + end + end + if #host_servers[host] == 0 then + host_servers[host] = nil + end + end + end + end +end + +-- helper to send HTTP Text Responses +local function sendTextResponse(txn, msg, status) + txn:set_status(status) + txn:add_header("Content-Type", "text/plain") + txn:start_response() + txn:send(msg) + return +end + +-- Stale Handler +local function removeStale() + local currentTime = core.now() + + for host, servers in pairs(host_servers) do + for i = #servers, 1, -1 do + local server = servers[i] + + if server.aliveUntil < currentTime.sec then + core.Info("Removed Server "..server.serverName.." from "..host..". Reason: Stale") + + table.remove(host_servers[host], i) + end + end + if #host_servers[host] == 0 then + host_servers[host] = nil + end + end +end + +function cleanup_stales() + while true do + core.msleep(host_server_stale_cleanup_interval * 1000) + + removeStale() + end +end + +-- register Stale-Cleanup task +core.register_task(cleanup_stales) + +-- HTTP endpoint handler +function handle_update(txn) + local body = txn:receive() + if not body or #body == 0 then + sendTextResponse(txn, "Empty body request", 400) + return + end + + local payload, err = json.decode(body) + if not payload then + sendTextResponse(txn, "Invalid JSON "..(err or "unknown"), 400) + return + end + + if not payload.serverName or not payload.diff then + sendTextResponse(txn, "Missing serverName or diff", 400) + return + end + + apply_diff(payload.serverName, payload.diff) + + sendTextResponse(txn, "Mapping successfully updated!", 200) + + core.Info("Mapping successfully updated!") +end + +-- register the HTTP endpoint +core.register_service("update_mapping", "http", handle_update) + +-- resolve backend for a given host +function resolve_backend(txn) + local host = txn:get_var("txn.sni") + + if host == nil or host == "" then + txn:set_var("txn.backend", "default_backend") + + core.Alert("No SNI found, using default backend") + + return + end + + local servers = host_servers[host] + + if not servers or #servers == 0 then + txn:set_var("txn.backend", "default_backend") + + core.Info("No Server found, using default backend") + elseif #servers == 1 then + txn:set_var("txn.backend", servers[1].serverName.."_backend") + + core.Info("Selected "..servers[1].serverName.." as Backend") + elseif #servers >= 1 then + local server_names = {} + for i, server in ipairs(servers) do + server_names[i] = server.serverName + end + + -- multiple servers -> failover backend + local backend_name = table.concat(server_names, "_") .. "_failover_backend" + txn:set_var("txn.backend", backend_name) + + core.Info("Selected "..backend_name.." as Failover Backend") + end +end + +core.register_action("resolve_backend", {"tcp-req", "http-req" }, resolve_backend) diff --git a/examples/whoami.docker-compose.yaml b/examples/whoami.docker-compose.yaml index 0f0e8cf..991ffc0 100644 --- a/examples/whoami.docker-compose.yaml +++ b/examples/whoami.docker-compose.yaml @@ -10,7 +10,7 @@ services: - traefik.http.routers.whoami.tls.certresolver=cloudflare - traefik.http.routers.whoami.service=whoami-svc - traefik.http.services.whoami-svc.loadbalancer.server.port=80 - # Enable Discovery on this Container + # Enable Discovery for this Container - discovery.enable=true networks: - traefik @@ -18,4 +18,3 @@ services: networks: traefik: external: true - diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..efb1876 --- /dev/null +++ b/go.mod @@ -0,0 +1,33 @@ +module github.com/codeshelldev/servdiscovery + +go 1.25.4 + +require github.com/codeshelldev/gotl v0.0.4 + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/moby/api v1.52.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + golang.org/x/sys v0.38.0 // indirect +) + +require ( + github.com/moby/moby/client v0.2.1 + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d6e4582 --- /dev/null +++ b/go.sum @@ -0,0 +1,47 @@ +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/codeshelldev/gotl v0.0.4 h1:W2cup8Pw9LzFLxmS5QUzY+NSE3ZgiRSUM7FiGd6qJrI= +github.com/codeshelldev/gotl v0.0.4/go.mod h1:Mfb+Lb+DV3DUXdA1sixJb2pLawaJGGFFeC29gUZQLcg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.52.0 h1:00BtlJY4MXkkt84WhUZPRqt5TvPbgig2FZvTbe3igYg= +github.com/moby/moby/api v1.52.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc= +github.com/moby/moby/client v0.2.1 h1:1Grh1552mvv6i+sYOdY+xKKVTvzJegcVMhuXocyDz/k= +github.com/moby/moby/client v0.2.1/go.mod h1:O+/tw5d4a1Ha/ZA/tPxIZJapJRUS6LNZ1wiVRxYHyUE= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/internals/config/config.go b/internals/config/config.go new file mode 100644 index 0000000..79647e1 --- /dev/null +++ b/internals/config/config.go @@ -0,0 +1,42 @@ +package config + +import ( + "os" + "strconv" + + "github.com/codeshelldev/gotl/pkg/logger" + "github.com/codeshelldev/servdiscovery/internals/config/structure" +) + +var ENV = &structure.ENV{ + DISCOVERY_INTERVAL: 60, + ALIVE_INTERVAL: 120, +} + +func Load() { + ENV.LOG_LEVEL = os.Getenv("LOG_LEVEL") + + discoveryInterval, err := strconv.Atoi(os.Getenv("DISCOVERY_INTERVAL")) + + if err != nil { + if discoveryInterval > 0 { + ENV.DISCOVERY_INTERVAL = discoveryInterval + } + } + + aliveInterval, err := strconv.Atoi(os.Getenv("ALIVE_INTERVAL")) + + if err != nil { + if aliveInterval > 0 { + ENV.ALIVE_INTERVAL = aliveInterval + } + } + + ENV.SERVER_NAME = os.Getenv("SERVER_NAME") + ENV.ENDPOINT = os.Getenv("ENDPOINT") + ENV.ENDPOINT_KEY = os.Getenv("ENDPOINT_KEY") +} + +func Log() { + logger.Dev("Loaded Environment:", ENV) +} \ No newline at end of file diff --git a/internals/config/structure/structure.go b/internals/config/structure/structure.go new file mode 100644 index 0000000..e9a96db --- /dev/null +++ b/internals/config/structure/structure.go @@ -0,0 +1,10 @@ +package structure + +type ENV struct { + LOG_LEVEL string + SERVER_NAME string + ENDPOINT string + ENDPOINT_KEY string + DISCOVERY_INTERVAL int + ALIVE_INTERVAL int +} \ No newline at end of file diff --git a/internals/discovery/diff.go b/internals/discovery/diff.go new file mode 100644 index 0000000..622e300 --- /dev/null +++ b/internals/discovery/diff.go @@ -0,0 +1,87 @@ +package discovery + +import ( + "strings" + + "github.com/codeshelldev/gotl/pkg/logger" +) + +type Diff[T any] struct { + Added []T + Removed []T +} + +func (diff *Diff[T]) Merge(other Diff[T]) { + diff.Added = append(diff.Added, other.Added...) + diff.Removed = append(diff.Removed, other.Removed...) +} + +func CleanDiff[T comparable](diff Diff[T]) Diff[T] { + removedMap := map[T]struct{}{} + addedMap := map[T]struct{}{} + + for _, r := range diff.Removed { + removedMap[r] = struct{}{} + } + for _, a := range diff.Added { + addedMap[a] = struct{}{} + } + + // Remove items that are in both + for removed := range removedMap { + _, exists := addedMap[removed] + if exists { + delete(removedMap, removed) + delete(addedMap, removed) + } + } + + cleaned := Diff[T]{} + + for removed := range removedMap { + cleaned.Removed = append(cleaned.Removed, removed) + } + for added := range addedMap { + cleaned.Added = append(cleaned.Added, added) + } + + return cleaned +} + +func logDiff(id string, diff Diff[string]) { + addedStr := strings.Join(diff.Added, ",") + removedStr := strings.Join(diff.Removed, ",") + + logger.Debug("[", id, "] ", "(+) ", addedStr, " (-) ", removedStr) +} + +func GetDiff[T comparable](old, new []T) Diff[T] { + diff := Diff[T]{} + + oldMap := map[T]struct{}{} + newMap := map[T]struct{}{} + + for _, value := range old { + oldMap[value] = struct{}{} + } + for _, value := range new { + newMap[value] = struct{}{} + } + + for value := range oldMap { + _, exists := newMap[value] + + if !exists { + diff.Removed = append(diff.Removed, value) + } + } + for value := range newMap { + _, exists := oldMap[value]; + + if !exists { + diff.Added = append(diff.Added, value) + } + } + + return diff +} \ No newline at end of file diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go new file mode 100644 index 0000000..1414d75 --- /dev/null +++ b/internals/discovery/discovery.go @@ -0,0 +1,225 @@ +package discovery + +import ( + "bytes" + "maps" + "net/http" + "regexp" + "slices" + "time" + + "github.com/codeshelldev/gotl/pkg/jsonutils" + "github.com/codeshelldev/gotl/pkg/logger" + "github.com/codeshelldev/servdiscovery/internals/docker" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" +) + +var containerHosts map[string][]string + +var containers []container.Summary + +func GetDiffDiscovery() Diff[string] { + logger.Debug("Starting discovery") + + diff, err := getContainerDiff() + + if err != nil { + logger.Error("Encountered error during discovery: ", err.Error()) + return Diff[string]{} + } + + logger.Debug("Cleaning diff") + + cleaned := CleanDiff(diff) + + return cleaned +} + +func GetAliveDiscovery() Diff[string] { + logger.Debug("Starting alive discovery") + + globalDiff := Diff[string]{ + Added: []string{}, + Removed: []string{}, + } + + newContainers, err := getEnabledContainers() + + if err != nil { + logger.Error("Encountered error during discovery: ", err.Error()) + return Diff[string]{} + } + + logger.Info("Found ", len(newContainers), " enabled containers") + + for _, container := range newContainers { + router := getRouterHosts(container) + + seq := maps.Values(router) + hostSlices := slices.Collect(seq) + hosts := slices.Concat(hostSlices...) + + globalDiff.Added = append(globalDiff.Added, hosts...) + } + + cleaned := CleanDiff(globalDiff) + + return cleaned +} + +func SendDiff(serverName, endpoint, key string, diff Diff[string]) (*http.Response, error) { + payload := map[string]any{ + "serverName": serverName, + "diff": diff, + } + + data, err := jsonutils.ToJsonSafe(payload) + if err != nil { + return nil, err + } + + client := &http.Client{ + Timeout: 10 * time.Second, + } + + req, err := http.NewRequest("POST", endpoint, bytes.NewReader([]byte(data))) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + + if key != "" { + req.Header.Set("Authorization", key) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return resp, nil +} + +func getContainerDiff() (Diff[string], error) { + globalDiff := Diff[string]{ + Added: []string{}, + Removed: []string{}, + } + + newContainers, err := getEnabledContainers() + + if err != nil { + return Diff[string]{}, err + } + + containerDiff := diffContainers(containers, newContainers) + + containers = newContainers + + logger.Info("Found ", len(containers), " enabled containers") + + logger.Debug("Found ", len(containerDiff.Added), " added containers") + logger.Debug("Found ", len(containerDiff.Removed), " removed containers") + + for _, container := range newContainers { + router := getRouterHosts(container) + + seq := maps.Values(router) + hostSlices := slices.Collect(seq) + hosts := slices.Concat(hostSlices...) + + old, exists := containerHosts[container.ID] + if exists { + diff := GetDiff(old, hosts) + + logDiff(container.Names[0], diff) + + globalDiff.Merge(diff) + } else { + logger.Info("Added ", container.Names[0]) + } + + containerHosts[container.ID] = hosts + } + + for _, removed := range containerDiff.Removed { + host, exists := containerHosts[removed.ID] + + if exists { + globalDiff.Removed = append(globalDiff.Removed, host...) + + logger.Info("Removed ", removed.Names[0]) + + delete(containerHosts, removed.ID) + } + } + + return globalDiff, nil +} + +func diffContainers(old, new []container.Summary) Diff[container.Summary] { + oldIDs := make([]string, 0, len(old)) + newIDs := make([]string, 0, len(new)) + + oldContainers := map[string]container.Summary{} + newContainers := map[string]container.Summary{} + + for _, container := range old { + oldIDs = append(oldIDs, container.ID) + oldContainers[container.ID] = container + } + for _, container := range new { + newIDs = append(newIDs, container.ID) + newContainers[container.ID] = container + } + + idDiff := GetDiff(oldIDs, newIDs) + + var diff Diff[container.Summary] + + for _, added := range idDiff.Added { + diff.Added = append(diff.Added, newContainers[added]) + } + for _, removed := range idDiff.Removed { + diff.Removed = append(diff.Removed, oldContainers[removed]) + } + + return diff +} + +func getRouterHosts(container container.Summary) map[string][]string { + hosts := map[string][]string{} + + hostRegex, err := regexp.Compile(`Host\(\x60([^\x60]+)\x60\)`) + + if err != nil { + return nil + } + + routerRegex, err := regexp.Compile(`traefik\.http\.routers\.([A-Za-z0-9._-]+)\.rule`) + + if err != nil { + return nil + } + + for key, value := range container.Labels { + router := routerRegex.FindString(key) + if router != "" { + hosts[router] = hostRegex.FindStringSubmatch(value) + } + } + + return hosts +} + +func getEnabledContainers() ([]container.Summary, error) { + filters := client.Filters{} + filters.Add("label", "discovery.enable=true") + + return docker.GetContainers(client.ContainerListOptions{ + Filters: filters, + }) +} \ No newline at end of file diff --git a/internals/docker/client.go b/internals/docker/client.go new file mode 100644 index 0000000..c7dc324 --- /dev/null +++ b/internals/docker/client.go @@ -0,0 +1,40 @@ +package docker + +import ( + "context" + "time" + + "github.com/codeshelldev/gotl/pkg/logger" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" +) + +var apiClient *client.Client + +func InitClient(options ...client.Opt) { + var err error + + if len(options) <= 0 { + options = append(options, client.WithHost("unix:///var/run/docker.sock")) + } + + apiClient, err = client.New(options...) + + if err != nil { + logger.Fatal("Could not connect to " + apiClient.DaemonHost() + ": ", err.Error()) + } + defer apiClient.Close() +} + +func GetContainers(options client.ContainerListOptions) ([]container.Summary, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + res, err := apiClient.ContainerList(ctx, options) + + if err != nil { + return []container.Summary{}, err + } + + return res.Items, nil +} \ No newline at end of file diff --git a/internals/docker/docker.go b/internals/docker/docker.go new file mode 100644 index 0000000..989c59c --- /dev/null +++ b/internals/docker/docker.go @@ -0,0 +1,30 @@ +package docker + +import ( + "os" + + "github.com/codeshelldev/gotl/pkg/docker" + log "github.com/codeshelldev/gotl/pkg/logger" +) + +func Init() { + log.Info("Running ", os.Getenv("IMAGE_TAG"), " Image") +} + +func Run(main func()) chan os.Signal { + return docker.Run(main) +} + +func Exit(code int) { + log.Info("Exiting...") + + docker.Exit(code) +} + +func Shutdown() { + log.Info("Shutdown signal received") + + log.Sync() + + log.Info("Server exited gracefully") +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..93aef9a --- /dev/null +++ b/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "sync" + "time" + + log "github.com/codeshelldev/gotl/pkg/logger" + "github.com/codeshelldev/servdiscovery/internals/config" + "github.com/codeshelldev/servdiscovery/internals/discovery" + "github.com/codeshelldev/servdiscovery/internals/docker" +) + +func main() { + config.Load() + + log.Init(config.ENV.LOG_LEVEL) + + log.Info("Initialized Logger with Level of ", log.Level()) + + if log.Level() == "dev" { + log.Dev("Welcome back Developer!") + } + + config.Log() + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + if config.ENV.DISCOVERY_INTERVAL <= 0 { + log.Info("Disabling diff discoveries") + return + } + + log.Debug("Started discovery loop") + + ticker := time.NewTicker(time.Duration(config.ENV.DISCOVERY_INTERVAL) * time.Second) + defer ticker.Stop() + + for range ticker.C { + process(discovery.GetDiffDiscovery()) + } + }() + + go func() { + defer wg.Done() + + if config.ENV.ALIVE_INTERVAL <= 0 { + log.Info("Disabling alive discoveries") + return + } + + log.Debug("Started alive discovery loop") + + ticker := time.NewTicker(time.Duration(config.ENV.ALIVE_INTERVAL) * time.Second) + defer ticker.Stop() + + for range ticker.C { + process(discovery.GetAliveDiscovery()) + } + }() + + stop := docker.Run(func() { + wg.Wait() + }) + + <-stop + docker.Shutdown() +} + +func process(diff discovery.Diff[string]) { + if len(diff.Added) <= 0 && len(diff.Removed) <= 0 { + log.Info("No changes detected, skipping...") + return + } + + log.Debug("Sending diff to ", config.ENV.ENDPOINT, " with ", "TOKEN:", config.ENV.ENDPOINT_KEY) + + resp, err := discovery.SendDiff(config.ENV.SERVER_NAME, config.ENV.ENDPOINT, config.ENV.ENDPOINT_KEY, diff) + + if err != nil { + log.Error("Error sending diff: ", err.Error()) + return + } + + log.Debug("Endpoint responded with ", resp.Status) +} From b45e72deda7dbb4c031e3a1817ea3bf4b7ae079d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:42:59 +0000 Subject: [PATCH 20/53] Update README.md --- README.md | 148 ++++++++++++++++++++++++------------------------------ 1 file changed, 66 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 8225881..2805eeb 100644 --- a/README.md +++ b/README.md @@ -1,129 +1,113 @@ +
+ + GitHub release + + + GitHub stars + + + Docker image size + + + Docker image Pulls + + + License: MIT + +
+ # ServDiscovery -ServDiscovery is a Discovery Service that keeps an Endpoint updated with active Hosts (of Services). +**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services—perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. ## Installation -> [!NOTE] -> ServDiscovery only works with Traefik and not with **any** other Reverse Proxy due to `discover.enable` lable +> [!IMPORTANT] +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discover.enable` label requirement. -Get the latest `docker-compose.yaml` file: +Get the latest `docker-compose.yaml`: ```yaml -services: - discovery: - image: ghcr.io/codeshelldev/servdiscovery:latest - container_name: service-discovery - environment: - ENDPOINT: https://mydomain.com/ENDPOINT - ENDPOINT_KEY: MY_VERY_SECURE_KEY - ALIVE_UPDATE_INTERVAL: 60 - SERVER_NAME: server-1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock +{{{ #://docker-compose.yaml }}} ``` +Then spin it up: + ```bash docker compose up -d ``` +Your discovery service is now live! 🎉 + ## Usage -Take this little `whoami` Container as an Example: +Let`s take a simple `whoami` container as an example: ```yaml -services: - whoami: - image: traefik/whoami:latest - container_name: whoami - labels: - - traefik.enable=true - - traefik.http.routers.whoami.rule=Host(`whoami.mydomain.com`) - - traefik.http.routers.whoami.entrypoints=websecure - - traefik.http.routers.whoami.tls=true - - traefik.http.routers.whoami.tls.certresolver=cloudflare - - traefik.http.routers.whoami.service=whoami-svc - - traefik.http.services.whoami-svc.loadbalancer.server.port=80 - # Enable Discovery on this Container - - discovery.enable=true - networks: - - traefik - -networks: - traefik: - external: true - +{{{ #://examples/whoami.docker-compose.yaml } } ``` -Whenever a new **Host-Rule** gets added / modified ServDiscovery will update the set Endpoint to notify of any new changes. -This way the Endpoint can correctly route to different Hosts based on **SNI / Hostnames**. +Whenever a new **Host-Rule** is added or updated, ServDiscovery will **automatically notify the configured endpoint**. +This ensures the endpoint can correctly route traffic based on **SNI / Hostnames**. -## Endpoint +## Endpoint Integration -ServDiscovery sends requests to the Endpoint as a **JSON HTTP Request**: +ServDiscovery communicates with your endpoint via **JSON HTTP Requests**: ```json -{ - "serverName": "server-1", - "diff": { - "added": [ - "whoami.mydomain.com", - "website.mydomain.com", - "auth.mydomain.com" - ], - "removed": [ - "whoami-backup.mydomain.com", - "website-backup.mydomain.com", - "auth-backup.mydomain.com" - ] - } -} +{{{ #://examples/payload.json }}} ``` -This example tells the Endpoint that... +Example explanation: -| Available | Unavailable | +| ✅ Available | ❌ Unavailable | | -------------------- | --------------------------- | | whoami.mydomain.com | whoami-backup.mydomain.com | | website.mydomain.com | website-backup.mydomain.com | | auth.mydomain.com | auth-backup.mydomain.com | -... is (un)available +This allows the endpoint (e.g., a load balancer) to remove `\*-backup` records from your registry and **update routable containers/services automatically**. -This way (if the Endpoint is used by a LoadBalancer) the Owner of the Endpoint can now delete the `*-backup.mydomain.com` records from a Registry, -thus updating the list of routable Containers / Services. +### Integrations -## Configuration +You can find example integrations inside of [examples/](./examples). -### ENDPOINT_KEY - -The Endpoint Key is provided in the Authorization Header (via Bearer) during the POST request between the Endpoint and ServDiscovery. -If no Key is provided ServDiscovery will leave out the Authorization Header. - -### DISCOVERY_INTERVAL +## Configuration -The Discovery Interval sets the Interval (in seconds) of which ServDiscovery will update the provided Endpoint. +### `ENDPOINT_KEY` -Default: `60` +The endpoint key is used in the `Authorization` header (Bearer token) when ServDiscovery sends POST requests. +If no key is provided, the header is omitted. -### FULL_DISCOVERY_INTERVAL +### `DISCOVERY_INTERVAL` -Sets the Interval for Full Disoveries. -This tells ServDiscovery to send a Full Update of all the activate Containers in the `added` JSON Key. +Time (in seconds) between updates to your endpoint. +**Default:** `60` seconds -> [!IMPORTANT] -> Must be set to a **fraction of** DISCOVERY_INTERVAL. -> (example: `DISCOVERY_INTERVAL * 2`) -> `FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL` must result in an int. -> `FULL_DISCOVERY_INTERVAL` must be bigger than `DISCOVERY_INTERVAL`. +### `ALIVE_INTERVAL` -Default: **Disabled** +Time (in seconds) between full alive discoveries. ServDiscovery sends a **complete update** of all active containers in the `added` JSON key. +**Default:** `120` seconds ## Contributing -Found a bug or have new ideas or enhancements for this Project? -Feel free to open up an issue or create a Pull Request! +Found a bug or have a brilliant idea? Contributions are welcome! Open an **issue** or create a **pull request** — your help makes this project better. ## License -[MIT](https://choosealicense.com/licenses/mit/) +This project is licensed under the [MIT License](./LICENSE). From 29bf913ff3eece69e0f11ecc5e27123a979142d2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:44:59 +0100 Subject: [PATCH 21/53] readme update --- .github/templates/README.template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 2805eeb..7c80cd0 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -33,7 +33,7 @@ # ServDiscovery -**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services—perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. +**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services — perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. ## Installation From e537c5453ff1d2b7e9cc7ac28bc72142c79b2adc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:45:29 +0000 Subject: [PATCH 22/53] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2805eeb..7c80cd0 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ # ServDiscovery -**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services—perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. +**ServDiscovery** is a dynamic **Discovery Service** that keeps your endpoints in sync with active hosts of your services — perfect for modern, containerized environments. Think of it as the bridge between your services and your reverse proxy, ensuring traffic always finds the right destination. ## Installation From cb1962390c45148aaa5d0ec07b5d500f3fac3e8e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:45:34 +0100 Subject: [PATCH 23/53] fix readme --- .github/templates/README.template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 7c80cd0..28ce032 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -38,7 +38,7 @@ ## Installation > [!IMPORTANT] -> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discover.enable` label requirement. +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discovery.enable` label requirement. Get the latest `docker-compose.yaml`: From 5b55dfb4b9aedf4c644dc5390cba607d7357d9f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:45:46 +0000 Subject: [PATCH 24/53] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c80cd0..28ce032 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ ## Installation > [!IMPORTANT] -> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discover.enable` label requirement. +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discovery.enable` label requirement. Get the latest `docker-compose.yaml`: From b194a9097093c95e4d523af3f4853d931f73b048 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:47:12 +0100 Subject: [PATCH 25/53] fix readme --- .github/templates/README.template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 28ce032..99b1c2c 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -56,7 +56,7 @@ Your discovery service is now live! 🎉 ## Usage -Let`s take a simple `whoami` container as an example: +Let's take a simple `whoami` container as an example: ```yaml {{{ #://examples/whoami.docker-compose.yaml } } From bf5e362a9da55e873939da89bb1ca58a0342c912 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:47:28 +0000 Subject: [PATCH 26/53] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 28ce032..99b1c2c 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Your discovery service is now live! 🎉 ## Usage -Let`s take a simple `whoami` container as an example: +Let's take a simple `whoami` container as an example: ```yaml {{{ #://examples/whoami.docker-compose.yaml } } From 9293c1b005c1f895bf9d4b80bacf88b496454c55 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:48:06 +0100 Subject: [PATCH 27/53] fix typo --- .github/templates/README.template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index 99b1c2c..a058123 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -59,7 +59,7 @@ Your discovery service is now live! 🎉 Let's take a simple `whoami` container as an example: ```yaml -{{{ #://examples/whoami.docker-compose.yaml } } +{{{ #://examples/whoami.docker-compose.yaml }}} ``` Whenever a new **Host-Rule** is added or updated, ServDiscovery will **automatically notify the configured endpoint**. From c61f5f1a7c09144e593be71fdd454475a66b1e56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:48:19 +0000 Subject: [PATCH 28/53] Update README.md --- README.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 99b1c2c..fdf2ab8 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,17 @@ Get the latest `docker-compose.yaml`: ```yaml -{{{ #://docker-compose.yaml }}} +services: + discovery: + image: ghcr.io/codeshelldev/servdiscovery:latest + container_name: service-discovery + environment: + ENDPOINT: https://mydomain.com/ENDPOINT + ENDPOINT_KEY: MY_VERY_SECURE_KEY + ALIVE_UPDATE_INTERVAL: 60 + SERVER_NAME: server-1 + volumes: + - /var/run/docker.sock:/var/run/docker.sock ``` Then spin it up: @@ -59,7 +69,26 @@ Your discovery service is now live! 🎉 Let's take a simple `whoami` container as an example: ```yaml -{{{ #://examples/whoami.docker-compose.yaml } } +services: + whoami: + image: traefik/whoami:latest + container_name: whoami + labels: + - traefik.enable=true + - traefik.http.routers.whoami.rule=Host(`whoami.mydomain.com`) + - traefik.http.routers.whoami.entrypoints=websecure + - traefik.http.routers.whoami.tls=true + - traefik.http.routers.whoami.tls.certresolver=cloudflare + - traefik.http.routers.whoami.service=whoami-svc + - traefik.http.services.whoami-svc.loadbalancer.server.port=80 + # Enable Discovery for this Container + - discovery.enable=true + networks: + - traefik + +networks: + traefik: + external: true ``` Whenever a new **Host-Rule** is added or updated, ServDiscovery will **automatically notify the configured endpoint**. @@ -70,7 +99,21 @@ This ensures the endpoint can correctly route traffic based on **SNI / Hostnames ServDiscovery communicates with your endpoint via **JSON HTTP Requests**: ```json -{{{ #://examples/payload.json }}} +{ + "serverName": "server-1", + "diff": { + "added": [ + "whoami.mydomain.com", + "website.mydomain.com", + "auth.mydomain.com" + ], + "removed": [ + "whoami-backup.mydomain.com", + "website-backup.mydomain.com", + "auth-backup.mydomain.com" + ] + } +} ``` Example explanation: From b6cdd5003d1ee2289f42bba0eb7b7cd5ebdf5140 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:52:57 +0100 Subject: [PATCH 29/53] update Dockerfile --- Dockerfile | 18 ++-- app.py | 283 ----------------------------------------------------- 2 files changed, 12 insertions(+), 289 deletions(-) delete mode 100644 app.py diff --git a/Dockerfile b/Dockerfile index 0b96b88..1251fea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,19 @@ -FROM python:3.12-alpine +FROM alpine:latest +RUN apk --no-cache add ca-certificates -WORKDIR /app +ARG IMAGE_TAG +ENV IMAGE_TAG=$IMAGE_TAG +LABEL org.opencontainers.image.version=$IMAGE_TAG + +ARG TARGETOS +ARG TARGETARCH -RUN pip install docker +WORKDIR /app COPY . . -ENV PORT=4531 +COPY dist/${TARGETOS}/${TARGETARCH}/app . -EXPOSE ${PORT} +RUN rm dist/ -r -CMD ["python", "app.py"] \ No newline at end of file +CMD ["./app"] \ No newline at end of file diff --git a/app.py b/app.py deleted file mode 100644 index 1a2b417..0000000 --- a/app.py +++ /dev/null @@ -1,283 +0,0 @@ -from docker import DockerClient -from urllib.parse import urlparse as parseUrl -import requests -import os -import signal -import logging -import re -import sys -from time import sleep - -# Threading -# import threading - -logger = logging.getLogger("ServDiscovery") - -handler = logging.StreamHandler(sys.stdout) -formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)s] %(message)s", - datefmt="%d.%m %H:%M" -) -handler.setFormatter(formatter) -logger.addHandler(handler) - -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") - - -SERVER_NAME = os.getenv("SERVER_NAME") - -ENDPOINT = os.getenv("ENDPOINT") -ENDPOINT_KEY = os.getenv("ENDPOINT_KEY") - -DISCOVERY_INTERVAL = os.getenv("DISCOVERY_INTERVAL") -FULL_DISCOVERY_INTERVAL = os.getenv("FULL_DISCOVERY_INTERVAL") - -fullDiscoveryRatio = 0 - -dockerClient = DockerClient(base_url="unix://var/run/docker.sock") - -# Threading -# containerToHostLock = threading.Lock() -# containerLock = threading.Lock() - -containersToHosts = {} -containers = {} - -def getDiff(old, new): - old_set, new_set = set(old), set(new) - - removed = old_set - new_set - added = new_set - old_set - - return { - "removed": list(removed), - "added": list(added) - } - -def getHostsFromLabels(labels): - hosts = [] - - for key, value in labels.items(): - if key.startswith("traefik.http.routers.") and key.endswith(".rule"): - matches = re.findall(r"Host\(`([^`]+)`\)", value) - hosts.extend(matches) - - return hosts - -def getEnabledContainers(): - newContainers = [] - - try: - newContainers = dockerClient.containers.list(filters={"label": "discovery.enable=true"}) - except Exception as e: - logger.error(f"Error fetching containers: {str(e)}") - - return newContainers - -def getDiscovery(): - diffs = { - "removed": [], - "added": [] - } - - global containers - - newContainers = getEnabledContainers() - - containerDiff = getDiff(containers, newContainers) - - logger.info(f"Found {newContainers.__len__()} Containers") - - logger.debug(f"Found {containerDiff.get("added").__len__()} added Containers") - logger.debug(f"Found {containerDiff.get("removed").__len__()} removed Containers") - - # Threading - # with containerToHostLock and containerLock: - - containers = newContainers - - # Update changed Containers and Add new Containers - for container in newContainers: - hosts = getHostsFromLabels(container.labels) - - if container.id in containersToHosts: - old = containersToHosts[container.id] - new = hosts - - # Get Difference - diff = getDiff(old, new) - - logger.debug(f"[{container.name}] + {diff.get("added")}, - {diff.get("removed")}") - - diffs.get("removed").extend(diff.get("removed")) - diffs.get("added").extend(diff.get("added")) - - containersToHosts[container.id] = hosts - else: - logger.debug(f"Added {container.name}") - - diffs.get("added").extend(hosts) - - containersToHosts[container.id] = hosts - - # Diff Old / Removed Containers - for removedContainer in containerDiff.get("removed"): - if removedContainer.id in containersToHosts: - - # Get all Hosts from Removed Container and Add them to the global Diff - diffs.get("removed").extend(containersToHosts[removedContainer.id]) - - # Remove Container from Dict - logger.debug(f"Removed {removedContainer.name}") - - containersToHosts.pop(removedContainer.id) - - return diffs - -def getFullDiscovery(): - diffs = { - "removed": [], - "added": [] - } - - newContainers = getEnabledContainers() - - logger.info(f"Found {newContainers.__len__()} Containers") - - for container in newContainers: - hosts = getHostsFromLabels(container.labels) - - diffs.get("added").extend(hosts) - - return diffs - -def cleanDiff(diff): - removed, added = set(diff.get("removed")), set(diff.get("added")) - - both = removed & added - - removed -= both - added -= both - - return { - "removed": list(removed), - "added": list(added) - } - -def exitContainer(): - logger.error(f"Shutting Container down...") - - os.kill(os.getpid(), signal.SIGTERM) - -def sendDiffToEndpoint(diff): - data = { "serverName": SERVER_NAME, "diff": diff } - - headers = {} - - if ENDPOINT_KEY: - headers["Authorization"] = f"Bearer {ENDPOINT_KEY}" - - response = requests.post( - url=ENDPOINT, - json=data, - headers=headers - ) - - return response - -# Threading -# def startBackgroundThread(): -# thread = threading.Thread(target=main, daemon=True) -# thread.start() - -def doDiscovery(): - logger.debug("Starting Discovery") - - globalDiff = getDiscovery() - - logger.debug("Cleaning Diff") - - return cleanDiff(globalDiff) - -def doFullDiscovery(): - logger.debug("Starting Full Discovery") - - globalDiff = getFullDiscovery() - - return globalDiff - -def main(): - i = 0 - while True: - isFullDiscovery = False - - if i == fullDiscoveryRatio: - isFullDiscovery = True - i = 0 - - i += 1 - - if not isFullDiscovery: - logger.info(f"Starting Discover in {DISCOVERY_INTERVAL}...") - else: - logger.info(f"Starting Full Discover in {DISCOVERY_INTERVAL}...") - - sleep(DISCOVERY_INTERVAL) - - globalDiff = { - "removed": [], - "added": [] - } - - if not isFullDiscovery: - globalDiff = doDiscovery() - else: - globalDiff = doFullDiscovery() - - # Check if there are any Changes - if globalDiff.get("removed").__len__() + globalDiff.get("added").__len__() <= 0: - logger.debug("No Changes were made, skipping...") - continue - - logger.info(f"Sending Diff to {ENDPOINT} with{"out" if not ENDPOINT_KEY else ""} Auth") - - response = sendDiffToEndpoint(globalDiff) - - logger.debug(f"Endpoint responded with {response.text} {response.status_code} {"OK" if response.ok else "NOT OK"}") - -if __name__ == '__main__': - logger.setLevel(level=LOG_LEVEL) - - if not SERVER_NAME or not ENDPOINT: - if not SERVER_NAME: - logger.error(f"No SERVER_NAME set") - if not ENDPOINT: - logger.error(f"No ENDPOINT set") - - exitContainer() - - if not DISCOVERY_INTERVAL: - logger.warning(f"No DISCOVERY_INTERVAL set, using 30sec as default") - DISCOVERY_INTERVAL = 30 - else: - DISCOVERY_INTERVAL = int(DISCOVERY_INTERVAL) - - if not FULL_DISCOVERY_INTERVAL: - logger.warning(f"No FULL_DISCOVERY_INTERVAL set, disregarding Full Discoveries") - FULL_DISCOVERY_INTERVAL = 0 - else: - FULL_DISCOVERY_INTERVAL = int(FULL_DISCOVERY_INTERVAL) - - if FULL_DISCOVERY_INTERVAL % DISCOVERY_INTERVAL != 0: - logger.warning(f"FULL_DISCOVERY_INTERVAL is not a valid Integer, rounding to nearest Int") - FULL_DISCOVERY_INTERVAL = round(FULL_DISCOVERY_INTERVAL) - - if FULL_DISCOVERY_INTERVAL != 0: - fullDiscoveryRatio = round(FULL_DISCOVERY_INTERVAL / DISCOVERY_INTERVAL) - logger.debug(f"FULL_DISCOVERY_INTERVAL ratio set to {fullDiscoveryRatio}") - - if not ENDPOINT_KEY or ENDPOINT_KEY == "": - logger.warning(f"No ENDPOINT_KEY set, requests may be denied") - - - main() From a3c59c1d5e782c7c63718fd54f06979640a2adc8 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:54:16 +0100 Subject: [PATCH 30/53] update workflows --- .github/workflows/docker-image-dev.yml | 7 ++++--- .github/workflows/docker-image.yml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-image-dev.yml b/.github/workflows/docker-image-dev.yml index 03e2292..513ef6c 100644 --- a/.github/workflows/docker-image-dev.yml +++ b/.github/workflows/docker-image-dev.yml @@ -4,12 +4,13 @@ on: push: branches: - dev - paths-ignore: - - ".**" + paths: + - "**/*.go" jobs: update: - uses: codeshelldev/gh-actions/.github/workflows/docker-image.yml@main + uses: codeshelldev/gh-actions/.github/workflows/docker-image-go.yml@main + name: Development Image with: registry: ghcr.io flavor: | diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d0c694f..25b5620 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -6,7 +6,8 @@ on: jobs: update: - uses: codeshelldev/gh-actions/.github/workflows/docker-image.yml@main + uses: codeshelldev/gh-actions/.github/workflows/docker-image-go.yml@main + name: Stable Image with: registry: ghcr.io secrets: From 032e200eff107edcd1845981fd48d5dba296475a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:56:04 +0100 Subject: [PATCH 31/53] update deps --- go.mod | 20 +++++++++++--------- go.sum | 12 ++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index efb1876..4f4ceb8 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module github.com/codeshelldev/servdiscovery -go 1.25.4 +go 1.25.5 -require github.com/codeshelldev/gotl v0.0.4 +require ( + github.com/codeshelldev/gotl v0.0.4 + github.com/moby/moby/api v1.52.0 +) require ( github.com/Microsoft/go-winio v0.6.2 // indirect @@ -12,17 +15,16 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/moby/api v1.52.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect golang.org/x/sys v0.38.0 // indirect ) diff --git a/go.sum b/go.sum index d6e4582..6098c01 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -31,14 +33,24 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= From 3e8e30cc549fb276bb16c4d429edeae8322bf898 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:56:42 +0100 Subject: [PATCH 32/53] pin alpine due to buildx issue --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1251fea..da53e45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:latest +FROM alpine:3.22 RUN apk --no-cache add ca-certificates ARG IMAGE_TAG From 63be50cdb2a461284bf69b986fc0cd8ad8a6e36b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:57:51 +0100 Subject: [PATCH 33/53] fix docker compose --- docker-compose.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ad61f32..80dd626 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,9 +3,10 @@ services: image: ghcr.io/codeshelldev/servdiscovery:latest container_name: service-discovery environment: - ENDPOINT: https://mydomain.com/ENDPOINT + ENDPOINT: https://mydomain.com/discover ENDPOINT_KEY: MY_VERY_SECURE_KEY - ALIVE_UPDATE_INTERVAL: 60 + DISCOVERY_INTERVAL: 60 + ALIVE_INTERVAL: 60 SERVER_NAME: server-1 volumes: - /var/run/docker.sock:/var/run/docker.sock From 12e3928cce8b1c77e0919f3614387272a75ac865 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:58:03 +0000 Subject: [PATCH 34/53] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fdf2ab8..8b4cb0d 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,10 @@ services: image: ghcr.io/codeshelldev/servdiscovery:latest container_name: service-discovery environment: - ENDPOINT: https://mydomain.com/ENDPOINT + ENDPOINT: https://mydomain.com/discover ENDPOINT_KEY: MY_VERY_SECURE_KEY - ALIVE_UPDATE_INTERVAL: 60 + DISCOVERY_INTERVAL: 60 + ALIVE_INTERVAL: 60 SERVER_NAME: server-1 volumes: - /var/run/docker.sock:/var/run/docker.sock From deb2231e13d5f49d5cea8fd53db8170a62bb7085 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:02:28 +0100 Subject: [PATCH 35/53] add log for docker image --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index 93aef9a..0d4122d 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,8 @@ func main() { log.Dev("Welcome back Developer!") } + docker.Init() + config.Log() var wg sync.WaitGroup From 710a6cc52317f83135c877de6f1913dee1288430 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:04:40 +0100 Subject: [PATCH 36/53] init docker socket client --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index 0d4122d..9066696 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,8 @@ func main() { config.Log() + docker.InitClient() + var wg sync.WaitGroup wg.Add(2) From b10fae95b97e9b02ac119eaa1ba431a95664cae1 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:06:37 +0100 Subject: [PATCH 37/53] fix readme --- .github/templates/README.template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/templates/README.template.md b/.github/templates/README.template.md index a058123..6d903c1 100644 --- a/.github/templates/README.template.md +++ b/.github/templates/README.template.md @@ -38,7 +38,7 @@ ## Installation > [!IMPORTANT] -> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discovery.enable` label requirement. +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to using traefik labels to determine routes. Get the latest `docker-compose.yaml`: From 6407d472f5019e6eeb4020aba33a2ce493f86cb1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:06:53 +0000 Subject: [PATCH 38/53] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b4cb0d..e8015fd 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ ## Installation > [!IMPORTANT] -> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to the `discovery.enable` label requirement. +> ServDiscovery works **only with Traefik**. It will **not** work with other reverse proxies due to using traefik labels to determine routes. Get the latest `docker-compose.yaml`: From ac2e11386ff6ebe9af23ae7225fe0603992717e5 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:08:01 +0100 Subject: [PATCH 39/53] rearrange logs --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 9066696..d122e65 100644 --- a/main.go +++ b/main.go @@ -15,14 +15,14 @@ func main() { log.Init(config.ENV.LOG_LEVEL) + docker.Init() + log.Info("Initialized Logger with Level of ", log.Level()) if log.Level() == "dev" { log.Dev("Welcome back Developer!") } - docker.Init() - config.Log() docker.InitClient() From 193094676115cd3d68a84a485c91ea7b81c8644b Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:09:22 +0100 Subject: [PATCH 40/53] initialize hostmap --- internals/discovery/discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index 1414d75..fa2bd02 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -15,7 +15,7 @@ import ( "github.com/moby/moby/client" ) -var containerHosts map[string][]string +var containerHosts = map[string][]string{} var containers []container.Summary From e84737216d0ee8cc076df008ae7cb81e9d31c8e2 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:14:41 +0100 Subject: [PATCH 41/53] hide logs when no changes --- internals/discovery/discovery.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index fa2bd02..f3b7cab 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -51,7 +51,7 @@ func GetAliveDiscovery() Diff[string] { return Diff[string]{} } - logger.Info("Found ", len(newContainers), " enabled containers") + logger.Debug("Found ", len(newContainers), " enabled containers") for _, container := range newContainers { router := getRouterHosts(container) @@ -121,8 +121,12 @@ func getContainerDiff() (Diff[string], error) { logger.Info("Found ", len(containers), " enabled containers") - logger.Debug("Found ", len(containerDiff.Added), " added containers") - logger.Debug("Found ", len(containerDiff.Removed), " removed containers") + if len(containerDiff.Added) > 0 { + logger.Debug("Found ", len(containerDiff.Added), " added containers") + } + if len(containerDiff.Removed) > 0 { + logger.Debug("Found ", len(containerDiff.Removed), " removed containers") + } for _, container := range newContainers { router := getRouterHosts(container) From e4c0ffc0c5ac061fd8f0000cd0a465795755a769 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:15:25 +0100 Subject: [PATCH 42/53] debug --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index d122e65..170beff 100644 --- a/main.go +++ b/main.go @@ -75,6 +75,8 @@ func main() { } func process(diff discovery.Diff[string]) { + log.Dev("Received diff: ", diff) + if len(diff.Added) <= 0 && len(diff.Removed) <= 0 { log.Info("No changes detected, skipping...") return From 11c52a9b2fc1994803da47a4c03146c8d2b08066 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:22:56 +0100 Subject: [PATCH 43/53] dont print diff if empty --- internals/discovery/diff.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internals/discovery/diff.go b/internals/discovery/diff.go index 622e300..fe069b7 100644 --- a/internals/discovery/diff.go +++ b/internals/discovery/diff.go @@ -49,6 +49,10 @@ func CleanDiff[T comparable](diff Diff[T]) Diff[T] { } func logDiff(id string, diff Diff[string]) { + if len(diff.Added) <= 0 && len(diff.Removed) <= 0 { + return + } + addedStr := strings.Join(diff.Added, ",") removedStr := strings.Join(diff.Removed, ",") From 1f0f397f5f4b9d9dbfc779d4994b0ea68d007e5e Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:24:05 +0100 Subject: [PATCH 44/53] cleanup --- go.sum | 34 +++++++++++++++++++++----------- internals/discovery/discovery.go | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/go.sum b/go.sum index 6098c01..7a806e1 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -15,12 +17,14 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/moby/api v1.52.0 h1:00BtlJY4MXkkt84WhUZPRqt5TvPbgig2FZvTbe3igYg= @@ -31,29 +35,35 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index f3b7cab..f45cabb 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -146,7 +146,7 @@ func getContainerDiff() (Diff[string], error) { logger.Info("Added ", container.Names[0]) } - containerHosts[container.ID] = hosts + containerHosts[container.ID] = hosts } for _, removed := range containerDiff.Removed { From d05208b7a47a3d64571872eca301604413658514 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:26:30 +0100 Subject: [PATCH 45/53] more debugging --- internals/discovery/discovery.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index f45cabb..9a4f5c4 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" "slices" + "strings" "time" "github.com/codeshelldev/gotl/pkg/jsonutils" @@ -144,6 +145,8 @@ func getContainerDiff() (Diff[string], error) { globalDiff.Merge(diff) } else { logger.Info("Added ", container.Names[0]) + + logger.Dev("!> With ", strings.Join(hosts, ",")) } containerHosts[container.ID] = hosts From da7c143401e8420a038457692cb9b796f4964918 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:33:38 +0100 Subject: [PATCH 46/53] fix host matching --- internals/discovery/discovery.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index 9a4f5c4..ea7cfaf 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -212,12 +212,20 @@ func getRouterHosts(container container.Summary) map[string][]string { return nil } - for key, value := range container.Labels { - router := routerRegex.FindString(key) - if router != "" { - hosts[router] = hostRegex.FindStringSubmatch(value) - } - } + for key, value := range container.Labels { + routerMatch := routerRegex.FindStringSubmatch(key) + if len(routerMatch) < 2 { + continue + } + router := routerMatch[1] + + matches := hostRegex.FindAllStringSubmatch(value, -1) + for _, match := range matches { + if len(match) >= 2 { + hosts[router] = append(hosts[router], match[1]) + } + } + } return hosts } From 55fe6a945a77e09836ed3b2f06d590d7a7e6f393 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:53:43 +0100 Subject: [PATCH 47/53] debugging --- internals/discovery/discovery.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index ea7cfaf..3f4dca0 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -30,10 +30,14 @@ func GetDiffDiscovery() Diff[string] { return Diff[string]{} } + logger.Dev("Raw: ", diff) + logger.Debug("Cleaning diff") cleaned := CleanDiff(diff) + logger.Dev("Cleaned: ", cleaned) + return cleaned } From aef37fcf8bacc3f2345c7bcdcfa6535a1f0c96a6 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 16:57:31 +0100 Subject: [PATCH 48/53] add name --- .github/workflows/readme-update.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/readme-update.yml b/.github/workflows/readme-update.yml index ed75786..8f55af2 100644 --- a/.github/workflows/readme-update.yml +++ b/.github/workflows/readme-update.yml @@ -10,5 +10,6 @@ on: jobs: update: uses: codeshelldev/gh-actions/.github/workflows/readme-update.yml@main + name: Update secrets: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 29aa32a64518755fafa7aa9347f08fb80377ff0f Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:04:18 +0100 Subject: [PATCH 49/53] actually add hosts for discoveries --- internals/discovery/discovery.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index 3f4dca0..3845c6a 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -150,6 +150,8 @@ func getContainerDiff() (Diff[string], error) { } else { logger.Info("Added ", container.Names[0]) + globalDiff.Added = append(globalDiff.Added, hosts...) + logger.Dev("!> With ", strings.Join(hosts, ",")) } From 49a100b1a10d6e33a8d474a7e7c20334ada24bd4 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:11:37 +0100 Subject: [PATCH 50/53] use map --- internals/discovery/discovery.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index 3845c6a..9eeb29a 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -76,7 +76,10 @@ func GetAliveDiscovery() Diff[string] { func SendDiff(serverName, endpoint, key string, diff Diff[string]) (*http.Response, error) { payload := map[string]any{ "serverName": serverName, - "diff": diff, + "diff": map[string]any{ + "added": diff.Added, + "removed": diff.Removed, + }, } data, err := jsonutils.ToJsonSafe(payload) From 16bf8134701c2aff1e9ccfa9cac962f85793ffe7 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:12:29 +0100 Subject: [PATCH 51/53] fix bearer auth --- internals/discovery/discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index 9eeb29a..d1681fc 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -99,7 +99,7 @@ func SendDiff(serverName, endpoint, key string, diff Diff[string]) (*http.Respon req.Header.Set("Content-Type", "application/json") if key != "" { - req.Header.Set("Authorization", key) + req.Header.Set("Authorization", "Bearer " + key) } resp, err := client.Do(req) From b015b00f15af628bede142ac2a384b1e27c52b4a Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:19:46 +0100 Subject: [PATCH 52/53] show if auth or not instead of leaking token --- main.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 170beff..ae66ef3 100644 --- a/main.go +++ b/main.go @@ -82,7 +82,13 @@ func process(diff discovery.Diff[string]) { return } - log.Debug("Sending diff to ", config.ENV.ENDPOINT, " with ", "TOKEN:", config.ENV.ENDPOINT_KEY) + withOrWithout := "out" + + if config.ENV.ENDPOINT_KEY != "" { + withOrWithout = "" + } + + log.Debug("Sending diff to ", config.ENV.ENDPOINT, " with", withOrWithout, " Auth") resp, err := discovery.SendDiff(config.ENV.SERVER_NAME, config.ENV.ENDPOINT, config.ENV.ENDPOINT_KEY, diff) From 38c0b8e64786bf5f8d8330d49202127fbb96deb9 Mon Sep 17 00:00:00 2001 From: CodeShell <122738806+CodeShellDev@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:20:54 +0100 Subject: [PATCH 53/53] remove devlogs --- internals/discovery/discovery.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internals/discovery/discovery.go b/internals/discovery/discovery.go index d1681fc..d12bc72 100644 --- a/internals/discovery/discovery.go +++ b/internals/discovery/discovery.go @@ -30,14 +30,10 @@ func GetDiffDiscovery() Diff[string] { return Diff[string]{} } - logger.Dev("Raw: ", diff) - logger.Debug("Cleaning diff") cleaned := CleanDiff(diff) - logger.Dev("Cleaned: ", cleaned) - return cleaned }