From 971371621debd8444e0a4a476d2c92596caa56cb Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 10:08:24 -0300 Subject: [PATCH 01/20] fix: add request_timeout to explorer --- explorer/lib/explorer/periodically.ex | 54 +++++++++++++------------ explorer/lib/explorer_web/live/utils.ex | 34 +++++++++------- 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 423b3738e6..54834cf2a2 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -79,32 +79,36 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - {batch_changeset, proofs} = - batch - |> Utils.extract_info_from_data_pointer() - |> Batches.generate_changesets() - - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") - - # no changes in DB - nil -> - nil + with {:ok, batch_info} <- Utils.extract_info_from_data_pointer(batch), + {batch_changeset, proofs} <- Batches.generate_changesets(batch_info) do + Batches.insert_or_update(batch_changeset, proofs) + |> case do + {:ok, _} -> + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + + {:error, error} -> + Logger.error( + "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" + ) + + # no changes in DB + nil -> + nil + end + + "Done processing batch: #{batch.merkle_root}" |> Logger.debug() + Mutex.release(BatchMutex, lock) + else + {:error, {:http_error, reason}} -> + Logger.error("Error when procesing request body: #{inspect(reason)}") + # Maybe delete the batch end - - "Done processing batch: #{batch.merkle_root}" |> Logger.debug() - Mutex.release(BatchMutex, lock) end end diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index ae4c503828..5145f8571d 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -168,19 +168,20 @@ defmodule Utils do end def calculate_proof_hashes({:ok, deserialized_batch}) do - deserialized_batch - |> Enum.map(fn s3_object -> - :crypto.hash(:sha3_256, s3_object["proof"]) - end) - end + proof_hashes = + deserialized_batch + |> Enum.map(fn s3_object -> + :crypto.hash(:sha3_256, s3_object["proof"]) + end) - def calculate_proof_hashes({:error, reason}) do - Logger.error("Error calculating proof hashes: #{inspect(reason)}") - [] + {:ok, proof_hashes} end + def calculate_proof_hashes(error), do: error + def fetch_batch_data_pointer(batch_data_pointer) do - case Finch.build(:get, batch_data_pointer) |> Finch.request(Explorer.Finch) do + case Finch.build(:get, batch_data_pointer) + |> Finch.request(Explorer.Finch, request_timeout: 10_000) do {:ok, %Finch.Response{status: 200, body: body}} -> cond do is_json?(body) -> @@ -240,7 +241,7 @@ defmodule Utils do def extract_info_from_data_pointer(%BatchDB{} = batch) do Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB - proof_hashes = + result = case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do nil -> Logger.debug("Fetching from S3") @@ -252,12 +253,17 @@ defmodule Utils do proof_hashes -> # already processed and stored the S3 data Logger.debug("Fetching from DB") - proof_hashes + {:ok, proof_hashes} end - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + with {:ok, proof_hashes} <- result do + batch_info = + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + + {:ok, batch_info} + end end def fetch_eigen_operator_metadata(url) do From ba51f1cbc29ec2f179606d5a4380ccb5c706213a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 10:13:48 -0300 Subject: [PATCH 02/20] refactor: replace the value with a constant --- explorer/lib/explorer_web/live/utils.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 5145f8571d..a31033bb2a 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -1,5 +1,7 @@ # Frontend Utils defmodule ExplorerWeb.Helpers do + @request_timeout 10_000 + def shorten_hash(hash, decimals \\ 6) do case String.length(hash) do n when n < decimals -> hash @@ -181,7 +183,7 @@ defmodule Utils do def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) - |> Finch.request(Explorer.Finch, request_timeout: 10_000) do + |> Finch.request(Explorer.Finch, request_timeout: @request_timeout) do {:ok, %Finch.Response{status: 200, body: body}} -> cond do is_json?(body) -> From 9e1069713350eea61764c4b9e6e4ca884cef260b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:19:19 -0300 Subject: [PATCH 03/20] refactor: handle error --- explorer/lib/explorer/periodically.ex | 63 ++++++++++++++------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 54834cf2a2..d1b54df381 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -16,8 +16,10 @@ defmodule Explorer.Periodically do one_second = 1000 seconds_in_an_hour = 60 * 60 - :timer.send_interval(one_second * 12, :batches) # every 12 seconds, once per block - :timer.send_interval(one_second * seconds_in_an_hour, :restakings) # every 1 hour + # every 12 seconds, once per block + :timer.send_interval(one_second * 12, :batches) + # every 1 hour + :timer.send_interval(one_second * seconds_in_an_hour, :restakings) end # Reads and process last blocks for operators and restaking changes @@ -45,6 +47,7 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) + if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -79,36 +82,34 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - with {:ok, batch_info} <- Utils.extract_info_from_data_pointer(batch), - {batch_changeset, proofs} <- Batches.generate_changesets(batch_info) do - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error( - "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" - ) - - # no changes in DB - nil -> - nil - end - - "Done processing batch: #{batch.merkle_root}" |> Logger.debug() - Mutex.release(BatchMutex, lock) - else - {:error, {:http_error, reason}} -> - Logger.error("Error when procesing request body: #{inspect(reason)}") - # Maybe delete the batch + {batch_changeset, proofs} = + batch + |> Utils.extract_info_from_data_pointer() + |> Batches.generate_changesets() + + Batches.insert_or_update(batch_changeset, proofs) + |> case do + {:ok, _} -> + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + + {:error, error} -> + Logger.error( + "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" + ) + + # no changes in DB + nil -> + nil end + + "Done processing batch: #{batch.merkle_root}" |> Logger.debug() + Mutex.release(BatchMutex, lock) end end From ae13563ebbaeee9c09aff511b231bc915a9f6df3 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:20:49 -0300 Subject: [PATCH 04/20] fix: adding missing changes --- explorer/lib/explorer_web/live/utils.ex | 59 +++++++++++++++---------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index a31033bb2a..375322383e 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -1,7 +1,5 @@ # Frontend Utils defmodule ExplorerWeb.Helpers do - @request_timeout 10_000 - def shorten_hash(hash, decimals \\ 6) do case String.length(hash) do n when n < decimals -> hash @@ -134,6 +132,8 @@ end # Backend utils defmodule Utils do require Logger + # 256 KB + @max_body_size 256 * 1024 def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -170,21 +170,40 @@ defmodule Utils do end def calculate_proof_hashes({:ok, deserialized_batch}) do - proof_hashes = - deserialized_batch - |> Enum.map(fn s3_object -> - :crypto.hash(:sha3_256, s3_object["proof"]) - end) + deserialized_batch + |> Enum.map(fn s3_object -> + :crypto.hash(:sha3_256, s3_object["proof"]) + end) + end + + def calculate_proof_hashes({:error, reason}) do + Logger.error("Error calculating proof hashes: #{inspect(reason)}") + [] + end + + defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} + defp stream_handler({:status, 200}, acc), do: {:cont, acc} - {:ok, proof_hashes} + defp stream_handler({:status, status_code}, _acc), + do: {:halt, {:error, {:http_error, status_code}}} + + defp stream_handler({:data, chunk}, {_acc_body, size}) + when size + byte_size(chunk) > @max_body_size do + {:halt, {:error, :body_too_large}} end - def calculate_proof_hashes(error), do: error + defp stream_handler({:data, chunk}, {acc_body, size}) do + new_size = size + byte_size(chunk) + {:cont, {acc_body <> chunk, new_size}} + end def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) - |> Finch.request(Explorer.Finch, request_timeout: @request_timeout) do - {:ok, %Finch.Response{status: 200, body: body}} -> + |> Finch.stream_while(Explorer.Finch, {"", 0}, &stream_handler(&1, &2)) do + {:ok, {:error, :body_too_large}} -> + {:error, {:http_error, :body_too_large}} + + {:ok, {body, _size}} -> cond do is_json?(body) -> case Jason.decode(body) do @@ -209,9 +228,6 @@ defmodule Utils do {:error, :unknown_format} end - {:ok, %Finch.Response{status: status_code}} -> - {:error, {:http_error, status_code}} - {:error, reason} -> {:error, {:http_error, reason}} end @@ -243,7 +259,7 @@ defmodule Utils do def extract_info_from_data_pointer(%BatchDB{} = batch) do Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB - result = + proof_hashes = case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do nil -> Logger.debug("Fetching from S3") @@ -255,17 +271,12 @@ defmodule Utils do proof_hashes -> # already processed and stored the S3 data Logger.debug("Fetching from DB") - {:ok, proof_hashes} + proof_hashes end - with {:ok, proof_hashes} <- result do - batch_info = - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) - - {:ok, batch_info} - end + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) end def fetch_eigen_operator_metadata(url) do From 6f6405a48da64dba00763f0933e00d8b29ce9935 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:24:57 -0300 Subject: [PATCH 05/20] refactor: fixing formatting issues --- explorer/lib/explorer/periodically.ex | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index d1b54df381..423b3738e6 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -16,10 +16,8 @@ defmodule Explorer.Periodically do one_second = 1000 seconds_in_an_hour = 60 * 60 - # every 12 seconds, once per block - :timer.send_interval(one_second * 12, :batches) - # every 1 hour - :timer.send_interval(one_second * seconds_in_an_hour, :restakings) + :timer.send_interval(one_second * 12, :batches) # every 12 seconds, once per block + :timer.send_interval(one_second * seconds_in_an_hour, :restakings) # every 1 hour end # Reads and process last blocks for operators and restaking changes @@ -47,7 +45,6 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) - if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -99,9 +96,7 @@ defmodule Explorer.Periodically do }) {:error, error} -> - Logger.error( - "Some error in DB operation, not broadcasting update_views: #{inspect(error)}" - ) + Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") # no changes in DB nil -> From 62c2fa5d61eb01c985c27677a3603cf37098707d Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:34:21 -0300 Subject: [PATCH 06/20] refactor: improve error msg --- explorer/lib/explorer_web/live/utils.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 375322383e..897adba183 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -189,7 +189,7 @@ defmodule Utils do defp stream_handler({:data, chunk}, {_acc_body, size}) when size + byte_size(chunk) > @max_body_size do - {:halt, {:error, :body_too_large}} + {:halt, {:error, {:http, :body_too_large}}} end defp stream_handler({:data, chunk}, {acc_body, size}) do @@ -200,8 +200,8 @@ defmodule Utils do def fetch_batch_data_pointer(batch_data_pointer) do case Finch.build(:get, batch_data_pointer) |> Finch.stream_while(Explorer.Finch, {"", 0}, &stream_handler(&1, &2)) do - {:ok, {:error, :body_too_large}} -> - {:error, {:http_error, :body_too_large}} + {:ok, {:error, reason}} -> + {:error, reason} {:ok, {body, _size}} -> cond do From d2519f043f7dc4719f88b0a2c3491a2bcc13905a Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 15:48:08 -0300 Subject: [PATCH 07/20] refactor: change constant to env --- explorer/.env.dev | 2 ++ explorer/lib/explorer_web/live/utils.ex | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/explorer/.env.dev b/explorer/.env.dev index c93bcea2c3..d594e549a1 100644 --- a/explorer/.env.dev +++ b/explorer/.env.dev @@ -20,3 +20,5 @@ DEBUG_ERRORS=true # Operator version tracker API TRACKER_API_URL=http://localhost:3030 + +MAX_BATCH_SIZE=268435456 # 256 MiB diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 897adba183..4332850ddb 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,8 +132,7 @@ end # Backend utils defmodule Utils do require Logger - # 256 KB - @max_body_size 256 * 1024 + @max_batch_size String.to_integer(System.get_env("POOL_SIZE") || "268435456") def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -188,7 +187,7 @@ defmodule Utils do do: {:halt, {:error, {:http_error, status_code}}} defp stream_handler({:data, chunk}, {_acc_body, size}) - when size + byte_size(chunk) > @max_body_size do + when size + byte_size(chunk) > @max_batch_size do {:halt, {:error, {:http, :body_too_large}}} end From 951eb28172dfa44e54c1d00aa44c44cc488c1b4c Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:00:23 -0300 Subject: [PATCH 08/20] fix: env variable --- explorer/lib/explorer_web/live/utils.ex | 2 +- explorer/start.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 4332850ddb..9fabfd9ab1 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,7 +132,7 @@ end # Backend utils defmodule Utils do require Logger - @max_batch_size String.to_integer(System.get_env("POOL_SIZE") || "268435456") + @max_batch_size String.to_integer(System.get_env("MAX_BATCH_SIZE") || "268435456") def string_to_bytes32(hex_string) do # Remove the '0x' prefix diff --git a/explorer/start.sh b/explorer/start.sh index 02d76fa6ad..94786e0112 100755 --- a/explorer/start.sh +++ b/explorer/start.sh @@ -14,6 +14,7 @@ env_vars=( "ALIGNED_CONFIG_FILE" "DEBUG_ERRORS" "TRACKER_API_URL" + "MAX_BATCH_SIZE" ) for var in "${env_vars[@]}"; do From 44552b3444239a0d115ec39e491fcc17e62da50b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 16:28:50 -0300 Subject: [PATCH 09/20] refactor: improve error handling --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 9fabfd9ab1..19e7656793 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -177,7 +177,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") - [] + ["invalid"] end defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} From 88aaca73d55245aa1ff5ba3edb9daa4bdf39ba57 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Tue, 8 Oct 2024 17:17:18 -0300 Subject: [PATCH 10/20] refactor: add comment --- explorer/lib/explorer_web/live/utils.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 19e7656793..474353c62f 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -177,6 +177,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") + # This ensures we avoid attempting to fetch the invalid data again. ["invalid"] end From e54d15b4360af6ad2e0125358151d584463459fe Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 00:57:34 -0300 Subject: [PATCH 11/20] refactor: error message --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 474353c62f..85c0ed3c9b 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -178,7 +178,7 @@ defmodule Utils do def calculate_proof_hashes({:error, reason}) do Logger.error("Error calculating proof hashes: #{inspect(reason)}") # This ensures we avoid attempting to fetch the invalid data again. - ["invalid"] + ["invalid batch"] end defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} From f2f3fcd8632aeaa08a1888eb1e9d338657ff2501 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:00:54 -0300 Subject: [PATCH 12/20] refactor: better error handling --- explorer/lib/explorer_web/live/utils.ex | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 85c0ed3c9b..070794d42c 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -175,12 +175,6 @@ defmodule Utils do end) end - def calculate_proof_hashes({:error, reason}) do - Logger.error("Error calculating proof hashes: #{inspect(reason)}") - # This ensures we avoid attempting to fetch the invalid data again. - ["invalid batch"] - end - defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} defp stream_handler({:status, 200}, acc), do: {:cont, acc} @@ -264,9 +258,17 @@ defmodule Utils do nil -> Logger.debug("Fetching from S3") - batch.data_pointer - |> Utils.fetch_batch_data_pointer() - |> Utils.calculate_proof_hashes() + batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + case batch_content do + {:ok, batch_content} -> + batch_content + |> Utils.calculate_proof_hashes() + + {:error, reason} -> + Logger.error("Error fetching batch content: #{inspect(reason)}") + # Returning something ensures we avoid attempting to fetch the invalid data again. + ["invalid batch"] + end proof_hashes -> # already processed and stored the S3 data From e69bc14bcae28a21d9c682565b0458df0ade2948 Mon Sep 17 00:00:00 2001 From: Uriel Mihura <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:08:57 -0300 Subject: [PATCH 13/20] Update explorer/lib/explorer_web/live/utils.ex --- explorer/lib/explorer_web/live/utils.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 070794d42c..f4d140e2d3 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,7 +132,11 @@ end # Backend utils defmodule Utils do require Logger - @max_batch_size String.to_integer(System.get_env("MAX_BATCH_SIZE") || "268435456") + @max_batch_size (case System.fetch_env("MAX_BATCH_SIZE") do + {:ok, ""} -> 268_435_456 #empty env var + {:ok, value} -> String.to_integer(value) + _ -> 268_435_456 # error + end) def string_to_bytes32(hex_string) do # Remove the '0x' prefix From 3875037af26b5a450d0a78f83d877edfbe44c623 Mon Sep 17 00:00:00 2001 From: Urix <43704209+uri-99@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:13:31 -0300 Subject: [PATCH 14/20] fix: calculate_proof_hashes --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index f4d140e2d3..4d0cf24efd 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -172,7 +172,7 @@ defmodule Utils do |> Enum.reverse() end - def calculate_proof_hashes({:ok, deserialized_batch}) do + def calculate_proof_hashes(deserialized_batch) do deserialized_batch |> Enum.map(fn s3_object -> :crypto.hash(:sha3_256, s3_object["proof"]) From 5cee221b884a13b3a9124d101b6fb8a2e7da1bee Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:31:39 -0300 Subject: [PATCH 15/20] refactor: check content lenght in the headers --- explorer/lib/explorer_web/live/utils.ex | 33 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 4d0cf24efd..c9fdba88d2 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -132,11 +132,14 @@ end # Backend utils defmodule Utils do require Logger + @max_batch_size (case System.fetch_env("MAX_BATCH_SIZE") do - {:ok, ""} -> 268_435_456 #empty env var - {:ok, value} -> String.to_integer(value) - _ -> 268_435_456 # error - end) + # empty env var + {:ok, ""} -> 268_435_456 + {:ok, value} -> String.to_integer(value) + # error + _ -> 268_435_456 + end) def string_to_bytes32(hex_string) do # Remove the '0x' prefix @@ -179,20 +182,27 @@ defmodule Utils do end) end - defp stream_handler({:headers, _headers}, acc), do: {:cont, acc} + defp stream_handler({:headers, headers}, acc) do + {_, batch_size} = List.keyfind(headers, "content-length", 0) + check_batch_size(batch_size, acc) + end + defp stream_handler({:status, 200}, acc), do: {:cont, acc} defp stream_handler({:status, status_code}, _acc), do: {:halt, {:error, {:http_error, status_code}}} - defp stream_handler({:data, chunk}, {_acc_body, size}) - when size + byte_size(chunk) > @max_batch_size do - {:halt, {:error, {:http, :body_too_large}}} + defp stream_handler({:data, chunk}, {acc_body, acc_size}) do + new_size = acc_size + byte_size(chunk) + check_batch_size(new_size, {acc_body <> chunk, new_size}) end - defp stream_handler({:data, chunk}, {acc_body, size}) do - new_size = size + byte_size(chunk) - {:cont, {acc_body <> chunk, new_size}} + defp check_batch_size(size, acc) do + if size > @max_batch_size do + {:halt, {:error, {:http, :body_too_large}}} + else + {:cont, acc} + end end def fetch_batch_data_pointer(batch_data_pointer) do @@ -263,6 +273,7 @@ defmodule Utils do Logger.debug("Fetching from S3") batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + case batch_content do {:ok, batch_content} -> batch_content From 9c5ef15677f8cf047cf4455c924415eed4834c9b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:35:44 -0300 Subject: [PATCH 16/20] fix: set invalid proof hashes to 0 --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index c9fdba88d2..694e35d228 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -282,7 +282,7 @@ defmodule Utils do {:error, reason} -> Logger.error("Error fetching batch content: #{inspect(reason)}") # Returning something ensures we avoid attempting to fetch the invalid data again. - ["invalid batch"] + ["0"] end proof_hashes -> From 8ab0e938cc1a765e6e2519438a9d8b5bc75bc2c0 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:46:49 -0300 Subject: [PATCH 17/20] fix: use a binary zero --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 694e35d228..8aeee8cfae 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -282,7 +282,7 @@ defmodule Utils do {:error, reason} -> Logger.error("Error fetching batch content: #{inspect(reason)}") # Returning something ensures we avoid attempting to fetch the invalid data again. - ["0"] + [<<0>>] end proof_hashes -> From 0466b3b04537c266a92a230512b6a237c683971b Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Wed, 9 Oct 2024 17:59:25 -0300 Subject: [PATCH 18/20] fix: set a default content length --- explorer/lib/explorer_web/live/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 8aeee8cfae..d4e5a08097 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,7 +183,7 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) check_batch_size(batch_size, acc) end From bcc9a52c8b4b123d85bc9b30d63032e0ad2d9b23 Mon Sep 17 00:00:00 2001 From: avilagaston9 Date: Thu, 10 Oct 2024 17:39:17 -0300 Subject: [PATCH 19/20] fix: convert content-length to integer --- explorer/lib/explorer_web/live/utils.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index d4e5a08097..048e363819 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -183,8 +183,8 @@ defmodule Utils do end defp stream_handler({:headers, headers}, acc) do - {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, 0}) - check_batch_size(batch_size, acc) + {_, batch_size} = List.keyfind(headers, "content-length", 0, {nil, "0"}) + check_batch_size(String.to_integer(batch_size), acc) end defp stream_handler({:status, 200}, acc), do: {:cont, acc} From 7e4c8c8a0435de265f5239326bad1606340e8dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Avila=20Gast=C3=B3n?= <72628438+avilagaston9@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:52:41 -0300 Subject: [PATCH 20/20] feat(explorer): add is_valid field to batches (#1212) Co-authored-by: Urix <43704209+uri-99@users.noreply.github.com> --- .../aligned_layer_service_manager.ex | 6 +- explorer/lib/explorer/models/batch_structs.ex | 6 +- explorer/lib/explorer/models/batches.ex | 12 +-- explorer/lib/explorer/periodically.ex | 33 ++++---- .../components/core_components.ex | 41 +++++++++- .../live/pages/batch/index.html.heex | 2 +- .../live/pages/batches/index.html.heex | 2 +- .../live/pages/operators/index.ex | 2 +- explorer/lib/explorer_web/live/utils.ex | 76 +++++++++++++------ .../20241010133259_add_is_valid_field.exs | 9 +++ 10 files changed, 131 insertions(+), 58 deletions(-) create mode 100644 explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs diff --git a/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex b/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex index fb31b19d50..bc276bba4b 100644 --- a/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex +++ b/explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex @@ -136,7 +136,8 @@ defmodule AlignedLayerServiceManager do proof_hashes: nil, fee_per_proof: BatcherPaymentServiceManager.get_fee_per_proof(%{merkle_root: created_batch.batchMerkleRoot}), sender_address: Utils.string_to_bytes32(created_batch.senderAddress), - max_aggregator_fee: created_batch.maxAggregatorFee + max_aggregator_fee: created_batch.maxAggregatorFee, + is_valid: true # set to false later if a process determines it is invalid } end @@ -166,7 +167,8 @@ defmodule AlignedLayerServiceManager do fee_per_proof: unverified_batch.fee_per_proof, proof_hashes: nil, sender_address: unverified_batch.sender_address, - max_aggregator_fee: unverified_batch.max_aggregator_fee + max_aggregator_fee: unverified_batch.max_aggregator_fee, + is_valid: true # set to false later if a process determines it is invalid } end end diff --git a/explorer/lib/explorer/models/batch_structs.ex b/explorer/lib/explorer/models/batch_structs.ex index 0ba1329d1a..599c0f0d13 100644 --- a/explorer/lib/explorer/models/batch_structs.ex +++ b/explorer/lib/explorer/models/batch_structs.ex @@ -32,7 +32,8 @@ defmodule BatchDB do :proof_hashes, :fee_per_proof, :sender_address, - :max_aggregator_fee + :max_aggregator_fee, + :is_valid ] defstruct [ :merkle_root, @@ -48,6 +49,7 @@ defmodule BatchDB do :proof_hashes, :fee_per_proof, :sender_address, - :max_aggregator_fee + :max_aggregator_fee, + :is_valid ] end diff --git a/explorer/lib/explorer/models/batches.ex b/explorer/lib/explorer/models/batches.ex index 7d2b430afa..f2474ef485 100644 --- a/explorer/lib/explorer/models/batches.ex +++ b/explorer/lib/explorer/models/batches.ex @@ -18,6 +18,7 @@ defmodule Batches do field :fee_per_proof, :integer field :sender_address, :binary field :max_aggregator_fee, :decimal + field :is_valid, :boolean, default: true timestamps() end @@ -25,8 +26,8 @@ defmodule Batches do @doc false def changeset(new_batch, updates) do new_batch - |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee]) - |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address]) + |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee, :is_valid]) + |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address, :is_valid]) |> validate_format(:merkle_root, ~r/0x[a-fA-F0-9]{64}/) |> unique_constraint(:merkle_root) |> validate_number(:amount_of_proofs, greater_than: 0) @@ -37,6 +38,7 @@ defmodule Batches do |> validate_format(:response_transaction_hash, ~r/0x[a-fA-F0-9]{64}/) |> validate_number(:max_aggregator_fee, greater_than: 0) |> validate_number(:fee_per_proof, greater_than_or_equal_to: 0) + |> validate_inclusion(:is_valid, [true, false]) end def cast_to_batches(%BatchDB{} = batch_db) do @@ -53,7 +55,8 @@ defmodule Batches do data_pointer: batch_db.data_pointer, fee_per_proof: batch_db.fee_per_proof, sender_address: batch_db.sender_address, - max_aggregator_fee: batch_db.max_aggregator_fee + max_aggregator_fee: batch_db.max_aggregator_fee, + is_valid: batch_db.is_valid } end @@ -113,7 +116,7 @@ defmodule Batches do threshold_datetime = DateTime.utc_now() |> DateTime.add(-43200, :second) # 12 hours ago query = from(b in Batches, - where: b.is_verified == false and b.submission_timestamp > ^threshold_datetime, + where: b.is_valid == true and b.is_verified == false and b.submission_timestamp > ^threshold_datetime, select: b) Explorer.Repo.all(query) @@ -193,5 +196,4 @@ defmodule Batches do end end end - end diff --git a/explorer/lib/explorer/periodically.ex b/explorer/lib/explorer/periodically.ex index 423b3738e6..403d651403 100644 --- a/explorer/lib/explorer/periodically.ex +++ b/explorer/lib/explorer/periodically.ex @@ -45,6 +45,7 @@ defmodule Explorer.Periodically do run_every_n_iterations = 8 new_count = rem(count + 1, run_every_n_iterations) + if new_count == 0 do Task.start(&process_unverified_batches/0) end @@ -79,25 +80,19 @@ defmodule Explorer.Periodically do {:ok, lock} -> "Processing batch: #{batch.merkle_root}" |> Logger.debug() - {batch_changeset, proofs} = - batch - |> Utils.extract_info_from_data_pointer() - |> Batches.generate_changesets() - - Batches.insert_or_update(batch_changeset, proofs) - |> case do - {:ok, _} -> - PubSub.broadcast(Explorer.PubSub, "update_views", %{ - eth_usd: - case EthConverter.get_eth_price_usd() do - {:ok, eth_usd_price} -> eth_usd_price - {:error, _error} -> :empty - end - }) - - {:error, error} -> - Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}") - + with {:ok, updated_batch} <- Utils.process_batch(batch), + {batch_changeset, proofs} <- Batches.generate_changesets(updated_batch), + {:ok, _} <- Batches.insert_or_update(batch_changeset, proofs) do + PubSub.broadcast(Explorer.PubSub, "update_views", %{ + eth_usd: + case EthConverter.get_eth_price_usd() do + {:ok, eth_usd_price} -> eth_usd_price + {:error, _error} -> :empty + end + }) + else + {:error, reason} -> + Logger.error("Error processing batch #{batch.merkle_root}. Error: #{inspect(reason)}") # no changes in DB nil -> nil diff --git a/explorer/lib/explorer_web/components/core_components.ex b/explorer/lib/explorer_web/components/core_components.ex index 54976b84f2..02a49e5653 100644 --- a/explorer/lib/explorer_web/components/core_components.ex +++ b/explorer/lib/explorer_web/components/core_components.ex @@ -417,15 +417,15 @@ defmodule ExplorerWeb.CoreComponents do end @doc """ - Renders a dynamic badge compoent. + Renders a dynamic badge component. """ attr :class, :string, default: nil attr :status, :boolean, default: true - attr :falsy_text, :string, default: "Pending" - attr :truthy_text, :string, default: "Verified" + attr :falsy_text, :string + attr :truthy_text, :string slot :inner_block, default: nil - def dynamic_badge(assigns) do + def dynamic_badge_boolean(assigns) do ~H""" <.badge variant={ @@ -449,6 +449,39 @@ defmodule ExplorerWeb.CoreComponents do """ end + @doc """ + Renders a dynamic badge component for the batcher. + """ + attr :class, :string, default: nil + attr :status, :atom + slot :inner_block, default: nil + + def dynamic_badge_for_batcher(assigns) do + ~H""" + <.badge + variant={ + case @status do + :invalid -> "destructive" + :verified -> "accent" + :pending -> "foreground" + end + } + class={ + classes([ + @class + ]) + } + > + <%= case @status do + :invalid -> "Invalid" + :verified -> "Verified" + :pending -> "Pending" + end %> + <%= render_slot(@inner_block) %> + + """ + end + @doc """ Renders an input with label and error messages. diff --git a/explorer/lib/explorer_web/live/pages/batch/index.html.heex b/explorer/lib/explorer_web/live/pages/batch/index.html.heex index 98636fec5e..764f12219c 100644 --- a/explorer/lib/explorer_web/live/pages/batch/index.html.heex +++ b/explorer/lib/explorer_web/live/pages/batch/index.html.heex @@ -25,7 +25,7 @@

Status:

- <.dynamic_badge class="w-fit" status={@current_batch.is_verified} /> + <.dynamic_badge_for_batcher class="w-fit" status={Helpers.get_batch_status(@current_batch)} />

diff --git a/explorer/lib/explorer_web/live/pages/batches/index.html.heex b/explorer/lib/explorer_web/live/pages/batches/index.html.heex index 044e43a14e..26ce5ac2ee 100644 --- a/explorer/lib/explorer_web/live/pages/batches/index.html.heex +++ b/explorer/lib/explorer_web/live/pages/batches/index.html.heex @@ -14,7 +14,7 @@ <:col :let={batch} label="Status"> - <.dynamic_badge status={batch.is_verified} /> + <.dynamic_badge_for_batcher status={Helpers.get_batch_status(batch)} /> <:col :let={batch} label="Age"> diff --git a/explorer/lib/explorer_web/live/pages/operators/index.ex b/explorer/lib/explorer_web/live/pages/operators/index.ex index 563cfce078..dab94a92e8 100644 --- a/explorer/lib/explorer_web/live/pages/operators/index.ex +++ b/explorer/lib/explorer_web/live/pages/operators/index.ex @@ -83,7 +83,7 @@ defmodule ExplorerWeb.Operators.Index do <%= operator.total_stake |> EthConverter.wei_to_eth(2) |> Helpers.format_number() %> ETH <:col :let={operator} label="Status"> - <.dynamic_badge status={operator.is_active} truthy_text="Active" falsy_text="Inactive" /> + <.dynamic_badge_boolean status={operator.is_active} truthy_text="Active" falsy_text="Inactive" /> <% else %> diff --git a/explorer/lib/explorer_web/live/utils.ex b/explorer/lib/explorer_web/live/utils.ex index 048e363819..270083825d 100644 --- a/explorer/lib/explorer_web/live/utils.ex +++ b/explorer/lib/explorer_web/live/utils.ex @@ -127,6 +127,14 @@ defmodule ExplorerWeb.Helpers do def binary_to_hex_string(binary) do Utils.binary_to_hex_string(binary) end + + def get_batch_status(batch) do + cond do + not batch.is_valid -> :invalid + batch.is_verified -> :verified + true -> :pending + end + end end # Backend utils @@ -199,7 +207,7 @@ defmodule Utils do defp check_batch_size(size, acc) do if size > @max_batch_size do - {:halt, {:error, {:http, :body_too_large}}} + {:halt, {:error, {:invalid, :body_too_large}}} else {:cont, acc} end @@ -233,7 +241,7 @@ defmodule Utils do true -> Logger.error("Unknown S3 object format") - {:error, :unknown_format} + {:error, {:invalid, :unknown_format}} end {:error, reason} -> @@ -264,36 +272,58 @@ defmodule Utils do end end - def extract_info_from_data_pointer(%BatchDB{} = batch) do + def process_batch(%BatchDB{} = batch) do + case get_proof_hashes(batch) do + {:ok, proof_hashes} -> + {:ok, add_proof_hashes_to_batch(batch, proof_hashes)} + + {:error, {:invalid, reason}} -> + Logger.error("Invalid batch content for #{batch.merkle_root}: #{inspect(reason)}") + # Returning something ensures we avoid attempting to fetch the invalid data again. + updated_batch = + batch + |> Map.put(:is_valid, false) + |> add_proof_hashes_to_batch([<<0>>]) + + {:ok, updated_batch} + + {:error, reason} -> + {:error, reason} + end + end + + defp add_proof_hashes_to_batch(batch, proof_hashes) do + batch + |> Map.put(:proof_hashes, proof_hashes) + |> Map.put(:amount_of_proofs, Enum.count(proof_hashes)) + end + + defp get_proof_hashes(%BatchDB{} = batch) do Logger.debug("Extracting batch's proofs info: #{batch.merkle_root}") # only get from s3 if not already in DB - proof_hashes = - case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do - nil -> - Logger.debug("Fetching from S3") + case Proofs.get_proofs_from_batch(%{merkle_root: batch.merkle_root}) do + nil -> + Logger.debug("Fetching from S3") - batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() + batch_content = batch.data_pointer |> Utils.fetch_batch_data_pointer() - case batch_content do - {:ok, batch_content} -> + case batch_content do + {:ok, batch_content} -> + proof_hashes = batch_content |> Utils.calculate_proof_hashes() - {:error, reason} -> - Logger.error("Error fetching batch content: #{inspect(reason)}") - # Returning something ensures we avoid attempting to fetch the invalid data again. - [<<0>>] - end + {:ok, proof_hashes} - proof_hashes -> - # already processed and stored the S3 data - Logger.debug("Fetching from DB") - proof_hashes - end + {:error, reason} -> + {:error, reason} + end - batch - |> Map.put(:proof_hashes, proof_hashes) - |> Map.put(:amount_of_proofs, proof_hashes |> Enum.count()) + proof_hashes -> + # already processed and stored the S3 data + Logger.debug("Fetching from DB") + {:ok, proof_hashes} + end end def fetch_eigen_operator_metadata(url) do diff --git a/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs new file mode 100644 index 0000000000..0864313352 --- /dev/null +++ b/explorer/priv/repo/migrations/20241010133259_add_is_valid_field.exs @@ -0,0 +1,9 @@ +defmodule Explorer.Repo.Migrations.AddIsValidField do + use Ecto.Migration + + def change do + alter table("batches") do + add :is_valid, :boolean, default: true + end + end +end