From 07a0e33e98b2e3730bead1d8fc69055cef162690 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 9 May 2025 21:13:37 +0200 Subject: [PATCH 1/4] fix(log): Use trace level for log output in hot-path Signed-off-by: Steffen Vogel --- lib/node.cpp | 8 ++++---- lib/nodes/can.cpp | 2 +- lib/nodes/infiniband.cpp | 2 +- lib/nodes/websocket.cpp | 4 ++-- lib/path_destination.cpp | 6 +++--- lib/path_source.cpp | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/node.cpp b/lib/node.cpp index 4a1633dc3..1b8982ea9 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -289,14 +289,14 @@ int Node::read(struct Sample *smps[], unsigned cnt) { if (stats != nullptr) stats->update(Stats::Metric::SMPS_SKIPPED, skipped); - logger->debug("Received {} samples of which {} have been skipped", nread, + logger->trace("Received {} samples of which {} have been skipped", nread, skipped); } else - logger->debug("Received {} samples", nread); + logger->trace("Received {} samples", nread); return rread; #else - logger->debug("Received {} samples", nread); + logger->trace("Received {} samples", nread); return nread; #endif // WITH_HOOKS @@ -330,7 +330,7 @@ int Node::write(struct Sample *smps[], unsigned cnt) { return sent; nsent += sent; - logger->debug("Sent {} samples", sent); + logger->trace("Sent {} samples", sent); } return nsent; diff --git a/lib/nodes/can.cpp b/lib/nodes/can.cpp index 2a35251c9..6e62551c5 100644 --- a/lib/nodes/can.cpp +++ b/lib/nodes/can.cpp @@ -400,7 +400,7 @@ int villas::node::can_read(NodeCompat *n, struct Sample *const smps[], if (!found_id) throw RuntimeError("Did not find signal for can id {}", frame.can_id); - n->logger->debug("Received {} signals", c->sample_buf_num); + n->logger->trace("Received {} signals", c->sample_buf_num); // Copy signal data to sample only when all signals have been received if (c->sample_buf_num == n->getInputSignals(false)->size()) { diff --git a/lib/nodes/infiniband.cpp b/lib/nodes/infiniband.cpp index 8b11ceaad..7897481fb 100644 --- a/lib/nodes/infiniband.cpp +++ b/lib/nodes/infiniband.cpp @@ -747,7 +747,7 @@ int villas::node::ib_read(NodeCompat *n, struct Sample *const smps[], // Get time directly after something arrived in Completion Queue ts_receive = time_now(); - n->logger->debug("Received {} Work Completions", wcs); + n->logger->trace("Received {} Work Completions", wcs); read_values = wcs; // Value to return max_wr_post = wcs; // Make space free in smps[] diff --git a/lib/nodes/websocket.cpp b/lib/nodes/websocket.cpp index 39da203fa..0caa1d358 100644 --- a/lib/nodes/websocket.cpp +++ b/lib/nodes/websocket.cpp @@ -271,7 +271,7 @@ int villas::node::websocket_protocol_cb(struct lws *wsi, if (ret < 0) return ret; - c->node->logger->debug("Send {} samples to connection: {}, bytes={}", + c->node->logger->trace("Sent {} samples to connection: {}, bytes={}", pulled, c->toString(), ret); } @@ -317,7 +317,7 @@ int villas::node::websocket_protocol_cb(struct lws *wsi, break; } - c->node->logger->debug("Received {} samples from connection: {}", recvd, + c->node->logger->trace("Received {} samples from connection: {}", recvd, c->toString()); // Set receive timestamp diff --git a/lib/path_destination.cpp b/lib/path_destination.cpp index c51332955..86d6c88cc 100644 --- a/lib/path_destination.cpp +++ b/lib/path_destination.cpp @@ -54,7 +54,7 @@ void PathDestination::enqueueAll(Path *p, const struct Sample *const smps[], // Increase reference counter of these samples as they are now also owned by the queue sample_incref_many(clones, cloned); - p->logger->debug("Enqueued {} samples to destination {} of path {}", + p->logger->trace("Enqueued {} samples to destination {} of path {}", enqueued, pd->node->getName(), p->toString()); } @@ -78,7 +78,7 @@ void PathDestination::write() { "Queue underrun for path {}: allocated={} expected={}", path->toString(), allocated, cnt); - path->logger->debug( + path->logger->trace( "Dequeued {} samples from queue of node {} which is part of path {}", allocated, node->getName(), path->toString()); @@ -93,7 +93,7 @@ void PathDestination::write() { int released = sample_decref_many(smps, allocated); - path->logger->debug("Released {} samples back to memory pool", released); + path->logger->trace("Released {} samples back to memory pool", released); } } diff --git a/lib/path_source.cpp b/lib/path_source.cpp index 05600a7bd..b1caafcfd 100644 --- a/lib/path_source.cpp +++ b/lib/path_source.cpp @@ -141,7 +141,7 @@ int PathSource::read(int i) { } else if (toenqueue != tomux) { int skipped = tomux - toenqueue; - path->logger->debug("Hooks skipped {} out of {} samples for path {}", + path->logger->trace("Hooks skipped {} out of {} samples for path {}", skipped, tomux, path->toString()); } #else From f5e943a41ea972a223afd5fc40e9f38efd89ecbc Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 9 May 2025 21:14:46 +0200 Subject: [PATCH 2/4] fix(node-webrtc): Improve debugging output Signed-off-by: Steffen Vogel --- lib/nodes/webrtc.cpp | 8 ++++++-- lib/nodes/webrtc/peer_connection.cpp | 11 +++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/nodes/webrtc.cpp b/lib/nodes/webrtc.cpp index b51033b9f..2c748eedb 100644 --- a/lib/nodes/webrtc.cpp +++ b/lib/nodes/webrtc.cpp @@ -185,8 +185,12 @@ std::vector WebRTCNode::getPollFDs() { } const std::string &WebRTCNode::getDetails() { - // TODO - details = fmt::format(""); + details = fmt::format("session={}, server={}, peer={}, wait_seconds={}, " + "max_retransmits={}, ordered={}, ice.tcp={}, ", + session, server, peer, wait_seconds, + *dci.reliability.maxRetransmits, + dci.reliability.unordered, rtcConf.enableIceTcp); + return details; } diff --git a/lib/nodes/webrtc/peer_connection.cpp b/lib/nodes/webrtc/peer_connection.cpp index 79e83dddd..3223ffb06 100644 --- a/lib/nodes/webrtc/peer_connection.cpp +++ b/lib/nodes/webrtc/peer_connection.cpp @@ -267,7 +267,7 @@ void PeerConnection::onSignalingDisconnected() { } void PeerConnection::onSignalingError(std::string err) { - logger->debug("Signaling connection error: {}", err); + logger->error("Signaling connection error: {}", err); auto lock = std::unique_lock{mutex}; @@ -281,12 +281,19 @@ void PeerConnection::onSignalingMessage(SignalingMessage msg) { std::visit( villas::utils::overloaded{ - [&](RelayMessage &c) { extraServers = std::move(c.servers); }, + [&](RelayMessage &c) { + for (auto &server : c.servers) { + logger->debug("Received ICE server: {}", server.hostname); + } + + extraServers = std::move(c.servers); + }, [&](ControlMessage &c) { auto const &id = c.peerID; if (c.peers.size() < 2) { + logger->warn("Not enough peers connected. Waiting for more."); resetConnectionAndStandby(lock); return; } From ed8533768bea4f630c595bad8356d8989bee7327 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 9 May 2025 21:15:12 +0200 Subject: [PATCH 3/4] fix(node-webrtc): Enable TCP by default Signed-off-by: Steffen Vogel --- lib/nodes/webrtc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nodes/webrtc.cpp b/lib/nodes/webrtc.cpp index 2c748eedb..e89026817 100644 --- a/lib/nodes/webrtc.cpp +++ b/lib/nodes/webrtc.cpp @@ -37,6 +37,8 @@ WebRTCNode::WebRTCNode(const uuid_t &id, const std::string &name) dci.reliability.maxRetransmits = 0; dci.reliability.unordered = true; #endif + + rtcConf.enableIceTcp = true; } WebRTCNode::~WebRTCNode() { @@ -110,8 +112,6 @@ int WebRTCNode::parse(json_t *json) { if (tcp > 0) rtcConf.enableIceTcp = tcp > 0; - else - rtcConf.enableIceTcp = true; } } From 200a566b979b5d3b01bafcad8b9f7ce658742d43 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Fri, 9 May 2025 21:16:08 +0200 Subject: [PATCH 4/4] feat(ci): Build RPM bundles and improve CI pipeline Signed-off-by: Steffen Vogel --- .gitlab-ci.yml | 126 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 34 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65108dbd3..a02ed3517 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ variables: DOCKER_CLI_EXPERIMENTAL: enabled CMAKE_BUILD_OPTS: "--parallel 16" CMAKE_EXTRA_OPTS: "-DCMAKE_BUILD_TYPE=Release -DVILLAS_COMPILE_WARNING_AS_ERROR=ON" + CACHIX_CACHE_NAME: villas stages: - prepare @@ -21,9 +22,41 @@ stages: - deploy - latest +# Anchors + +.nix: &nix + image: docker.nix-community.org/nixpkgs/cachix-flakes + + parallel: + matrix: + - SYSTEM: [x86_64-linux, aarch64-linux] + + before_script: + - cachix use "$CACHIX_CACHE_NAME" + +.nix_packaging: &nix_packaging + <<: *nix + + stage: packaging + needs: + - job: build:nix + parallel: + matrix: + - SYSTEM: [x86_64-linux, aarch64-linux] + +.nix_packaging_artifacts: &nix_packaging_artifacts + <<: *nix_packaging + + artifacts: + when: on_success + access: all + expire_in: "1 year" + paths: + - artifacts/* + # Stage: prepare -# Build docker image which is used to build & test VILLASnode +# Build Docker image which is used to build & test VILLASnode prepare:docker: stage: prepare before_script: @@ -88,42 +121,16 @@ build:source: -DCMAKE_PREFIX_PATH=/usr/local build:nix: + <<: *nix + stage: build - image: docker.nix-community.org/nixpkgs/cachix-flakes needs: [] - variables: - CACHIX_CACHE_NAME: villas - - parallel: - matrix: - - SYSTEM: [x86_64-linux, aarch64-linux] - - artifacts: - when: on_success - access: all - expire_in: "1 year" - paths: - - artifacts/* - - before_script: - - cachix use "$CACHIX_CACHE_NAME" - - - | # Create directories - mkdir -p artifacts /var/tmp/ - - - | # Login at Docker registry - nix run nixpkgs#skopeo -- login \ - --username ${CI_REGISTRY_USER} \ - --password ${CI_REGISTRY_PASSWORD} \ - ${CI_REGISTRY} script: - - cachix watch-exec $CACHIX_CACHE_NAME -- nix build --print-build-logs ".#villas-node-${SYSTEM}" - - cachix watch-exec $CACHIX_CACHE_NAME -- nix bundle --print-build-logs --out-link bundle ".#villas-node-${SYSTEM}" - - cachix watch-exec $CACHIX_CACHE_NAME -- nix build --print-build-logs --out-link docker ".#dockerImage-${SYSTEM}" - - - cp -L bundle artifacts/villas-${SYSTEM} - - nix run nixpkgs#skopeo -- --tmpdir=${TMPDIR} --insecure-policy copy "docker-archive:./docker" "docker://${DOCKER_IMAGE}:${DOCKER_TAG}-nix-${SYSTEM}" + - cachix watch-exec $CACHIX_CACHE_NAME -- + nix build + --print-build-logs + ".#villas-node-${SYSTEM}" # Stage: test @@ -207,6 +214,7 @@ test:reuse: pkg:docker: stage: packaging + needs: [] before_script: - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} script: @@ -236,7 +244,57 @@ pkg:docker: ARCH: arm64 TRIPLET: aarch64-linux-gnu TAG: pi5 - needs: [] + +pkg:nix:arx: + <<: *nix_packaging_artifacts + + script: + - cachix watch-exec $CACHIX_CACHE_NAME -- + nix bundle + --print-build-logs + --out-link bundle-arx + --bundler github:NixOS/bundlers#toArx + ".#villas-node-${SYSTEM}" + + - mkdir -p artifacts + - cp -L bundle-arx artifacts/villas-${SYSTEM} + +pkg:nix:rpm: + <<: *nix_packaging_artifacts + + script: + - cachix watch-exec $CACHIX_CACHE_NAME -- + nix bundle + --print-build-logs + --out-link bundle-rpm + --bundler github:NixOS/bundlers#toRPM + ".#villas-node-${SYSTEM}" + + - mkdir -p artifacts + - cp -L bundle-rpm/*.rpm artifacts/villas-${SYSTEM}.rpm + +pkg:nix:docker: + <<: *nix_packaging + + script: + - mkdir -p /var/tmp/ + - cachix watch-exec $CACHIX_CACHE_NAME -- + nix bundle + --print-build-logs + --out-link bundle-docker + --bundler github:NixOS/bundlers#toDockerImage + ".#villas-node-${SYSTEM}" + + - nix run nixpkgs#skopeo -- + login + --username ${CI_REGISTRY_USER} + --password ${CI_REGISTRY_PASSWORD} + ${CI_REGISTRY} + + - nix run nixpkgs#skopeo -- + --tmpdir=${TMPDIR} + --insecure-policy + copy "docker-archive:./bundle-docker" "docker://${DOCKER_IMAGE}:${DOCKER_TAG}-nix-${SYSTEM}" # Stage: deploy