From b0047dcb16426f4d7e3a545e8585e020a25f752a Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 14:20:30 -0600 Subject: [PATCH 1/6] Issue #137 conform to message, batch and queue size requirements and messaging --- lib/Segment/Consumer/ForkCurl.php | 6 ++---- lib/Segment/Consumer/LibCurl.php | 6 ++---- lib/Segment/Consumer/Socket.php | 6 ++---- lib/Segment/QueueConsumer.php | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/Segment/Consumer/ForkCurl.php b/lib/Segment/Consumer/ForkCurl.php index 6f73a00..c7b1f9e 100644 --- a/lib/Segment/Consumer/ForkCurl.php +++ b/lib/Segment/Consumer/ForkCurl.php @@ -69,10 +69,8 @@ public function flushBatch($messages) { // Verify message size is below than 32KB if (strlen($payload) >= 32 * 1024) { - if ($this->debug()) { - $msg = "Message size is larger than 32KB"; - error_log("[Analytics][" . $this->type . "] " . $msg); - } + $msg = "Message size is larger than 32KB"; + error_log("[Analytics][" . $this->type . "] " . $msg); return false; } diff --git a/lib/Segment/Consumer/LibCurl.php b/lib/Segment/Consumer/LibCurl.php index 842a803..052f4bb 100644 --- a/lib/Segment/Consumer/LibCurl.php +++ b/lib/Segment/Consumer/LibCurl.php @@ -34,10 +34,8 @@ public function flushBatch($messages) { // Verify message size is below than 32KB if (strlen($payload) >= 32 * 1024) { - if ($this->debug()) { - $msg = "Message size is larger than 32KB"; - error_log("[Analytics][" . $this->type . "] " . $msg); - } + $msg = "Message size is larger than 32KB"; + error_log("[Analytics][" . $this->type . "] " . $msg); return false; } diff --git a/lib/Segment/Consumer/Socket.php b/lib/Segment/Consumer/Socket.php index 2bec79b..ae58e0f 100644 --- a/lib/Segment/Consumer/Socket.php +++ b/lib/Segment/Consumer/Socket.php @@ -193,10 +193,8 @@ private function createBody($host, $content) { // Verify message size is below than 32KB if (strlen($req) >= 32 * 1024) { - if ($this->debug()) { - $msg = "Message size is larger than 32KB"; - error_log("[Analytics][" . $this->type . "] " . $msg); - } + $msg = "Message size is larger than 32KB"; + error_log("[Analytics][" . $this->type . "] " . $msg); return false; } diff --git a/lib/Segment/QueueConsumer.php b/lib/Segment/QueueConsumer.php index 682950f..27d7edf 100644 --- a/lib/Segment/QueueConsumer.php +++ b/lib/Segment/QueueConsumer.php @@ -5,7 +5,10 @@ abstract class Segment_QueueConsumer extends Segment_Consumer { protected $queue; protected $max_queue_size = 1000; + protected $max_queue_size_bytes = 33,554,432; //32M + protected $batch_size = 100; + protected $max_batch_size_bytes = 512000; //500kb protected $maximum_backoff_duration = 10000; // Set maximum waiting limit to 10s protected $host = ""; protected $compress_request = false; @@ -111,6 +114,14 @@ public function flush() { while ($count > 0 && $success) { $batch = array_splice($this->queue, 0, min($this->batch_size, $count)); + + if (strlen($batch) >= $max_batch_size_bytes) { + $msg = "Batch size is larger than 500KB"; + error_log("[Analytics][" . $this->type . "] " . $msg); + + return false; + } + $success = $this->flushBatch($batch); $count = count($this->queue); @@ -131,7 +142,15 @@ protected function enqueue($item) { return false; } + if (strlen($this->queue) >= $max_queue_size_bytes) { + $msg = "Queue size is larger than 32MB"; + error_log("[Analytics][" . $this->type . "] " . $msg); + + return false; + } + $count = array_push($this->queue, $item); + if ($count >= $this->batch_size) { return $this->flush(); // return ->flush() result: true on success From 2e4983939c275b05c35db56b93d3126757470ca9 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 14:28:35 -0600 Subject: [PATCH 2/6] Issue #137: Resolve syntax error in number. --- lib/Segment/QueueConsumer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Segment/QueueConsumer.php b/lib/Segment/QueueConsumer.php index 27d7edf..ee4c1dd 100644 --- a/lib/Segment/QueueConsumer.php +++ b/lib/Segment/QueueConsumer.php @@ -5,7 +5,7 @@ abstract class Segment_QueueConsumer extends Segment_Consumer { protected $queue; protected $max_queue_size = 1000; - protected $max_queue_size_bytes = 33,554,432; //32M + protected $max_queue_size_bytes = 33554432; //32M protected $batch_size = 100; protected $max_batch_size_bytes = 512000; //500kb From 8daa3af96b6288e1e34c15ea06cd69c554e73ef6 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 14:42:15 -0600 Subject: [PATCH 3/6] Issue #137: Update size test to accommodate byte size of arrays --- lib/Segment/QueueConsumer.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Segment/QueueConsumer.php b/lib/Segment/QueueConsumer.php index ee4c1dd..062d3a1 100644 --- a/lib/Segment/QueueConsumer.php +++ b/lib/Segment/QueueConsumer.php @@ -115,7 +115,7 @@ public function flush() { while ($count > 0 && $success) { $batch = array_splice($this->queue, 0, min($this->batch_size, $count)); - if (strlen($batch) >= $max_batch_size_bytes) { + if (mb_strlen(serialize((array)$this->queue), '8bit') >= $this->max_batch_size_bytes) { $msg = "Batch size is larger than 500KB"; error_log("[Analytics][" . $this->type . "] " . $msg); @@ -142,7 +142,7 @@ protected function enqueue($item) { return false; } - if (strlen($this->queue) >= $max_queue_size_bytes) { + if (mb_strlen(serialize((array)$this->queue), '8bit') >= $this->max_queue_size_bytes) { $msg = "Queue size is larger than 32MB"; error_log("[Analytics][" . $this->type . "] " . $msg); From af9f3aa79bd4f1e8aa64a0572b57ca052dba40ad Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 16:07:05 -0600 Subject: [PATCH 4/6] Issue #142 update e2e to version 0.4.1-pre1 --- .buildscript/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildscript/e2e.sh b/.buildscript/e2e.sh index ed25afe..5ccdab4 100755 --- a/.buildscript/e2e.sh +++ b/.buildscript/e2e.sh @@ -6,7 +6,7 @@ if [ "$RUN_E2E_TESTS" != "true" ]; then echo "Skipping end to end tests." else echo "Running end to end tests..." - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64 -O tester + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.4.1-pre1/tester_linux_amd64 -O tester chmod +x tester ./tester -path='./bin/analytics' echo "End to end tests completed!" From 797543d333a6a79ed1aa3439684e70cee27efd61 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 16:38:07 -0600 Subject: [PATCH 5/6] Revert "Issue #142 update e2e to version 0.4.1-pre1" This reverts commit af9f3aa79bd4f1e8aa64a0572b57ca052dba40ad. --- .buildscript/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildscript/e2e.sh b/.buildscript/e2e.sh index 5ccdab4..ed25afe 100755 --- a/.buildscript/e2e.sh +++ b/.buildscript/e2e.sh @@ -6,7 +6,7 @@ if [ "$RUN_E2E_TESTS" != "true" ]; then echo "Skipping end to end tests." else echo "Running end to end tests..." - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.4.1-pre1/tester_linux_amd64 -O tester + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64 -O tester chmod +x tester ./tester -path='./bin/analytics' echo "End to end tests completed!" From cd051eac727cb84ef8953b712e00d997446f1dd1 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Sun, 1 Nov 2020 16:41:52 -0600 Subject: [PATCH 6/6] Issue #142 update e2e to version 0.4.1-pre1 --- .buildscript/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildscript/e2e.sh b/.buildscript/e2e.sh index ed25afe..5ccdab4 100755 --- a/.buildscript/e2e.sh +++ b/.buildscript/e2e.sh @@ -6,7 +6,7 @@ if [ "$RUN_E2E_TESTS" != "true" ]; then echo "Skipping end to end tests." else echo "Running end to end tests..." - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64 -O tester + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.4.1-pre1/tester_linux_amd64 -O tester chmod +x tester ./tester -path='./bin/analytics' echo "End to end tests completed!"