feat: restructure RPC service

infrastructure/nomad/playbooks/templates/jobs/preconf-rpc.nomad.j2

@@ -42,9 +42,119 @@ job "{{ job.name }}" {
       port = "{{ port_name }}"
       tags = ["{{ port_name }}"]
       provider = "nomad"
+      {% if port_name == "http" %}
+      check {
+        type     = "http"
+        path     = "/health"
+        interval = "10s"
+        timeout  = "2s"
+      }
+      {% endif %}
     }
     {% endfor %}
 
+    task "db" {
+      driver = "exec"
+
+      lifecycle {
+        hook    = "prestart"
+        sidecar = true
+      }
+
+      {% if profile == 'testnet' or profile == 'mainnet' %}
+      resources {
+        cores = 4
+        memory = 8192
+      }
+      {% elif profile == 'stressnet' or profile == 'stressnet-wl1' %}
+      resources {
+        memory = 4096
+      }
+      {% endif %}
+
+      template {
+        data = <<-EOH
+          POSTGRES_VERSION="15"
+          POSTGRES_DB="preconf-rpc"
+          POSTGRES_USERNAME="preconf-rpc"
+          POSTGRES_PASSWORD="{{ lookup('password', '/dev/null', length=64) }}"
+          {%- raw %}
+          POSTGRES_DATA="/local/pgdata-{{ env "NOMAD_ALLOC_INDEX" }}"
+          {{- range nomadService "{% endraw %}{{ job.name }}{% raw %}" }}
+            {{- if contains "db" .Tags }}
+          POSTGRES_PORT="{{ .Port }}"
+            {{- end }}
+          {{- end }}
+          {% endraw %}
+        EOH
+        destination = "alloc/data/postgres.env"
+        env = true
+      }
+
+      template {
+        data = <<-EOH
+          #!/usr/bin/env bash
+
+          {% raw %}
+          {{- range nomadService "datadog-agent-logs-collector" }}
+            {{ if contains "tcp" .Tags }}
+          exec > >(nc {{ .Address }} {{ .Port }}) 2>&1
+            {{ end }}
+          {{- end }}
+
+          if [ -d "${POSTGRES_DATA}" ]; then
+            echo "Initialized and configured database found."
+            cp "${POSTGRES_DATA}/.env" /alloc/data/postgres.env
+            source "${POSTGRES_DATA}/.env"
+            postgres -D ${POSTGRES_DATA}
+            exit $?
+          fi
+
+          export PATH="/usr/lib/postgresql/${POSTGRES_VERSION}/bin:${PATH}"
+          mkdir -p /var/run/postgresql > /dev/null 2>&1
+          pg_ctl initdb --silent --pgdata="${POSTGRES_DATA}"
+          if [ $? -ne 0 ]; then
+            echo "Failed to initialize PostgreSQL."
+            exit 1
+          fi
+          cp /alloc/data/postgres.env "${POSTGRES_DATA}/.env"
+
+          pg_ctl start --pgdata="${POSTGRES_DATA}" --silent --wait --timeout=300 > /dev/null 2>&1
+          if [ $? -ne 0 ]; then
+            echo "Failed to start PostgreSQL."
+            exit 1
+          fi
+
+          createuser --superuser postgres > /dev/null 2>&1
+          createuser --username=postgres --createdb "${POSTGRES_USERNAME}"
+          createdb --username="${POSTGRES_USERNAME}" "${POSTGRES_DB}"
+          psql --quiet \
+               --username="${POSTGRES_USERNAME}" \
+               --dbname="${POSTGRES_DB}" \
+               --command="ALTER USER ${POSTGRES_USERNAME} WITH PASSWORD '${POSTGRES_PASSWORD}'; \
+                          GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USERNAME};"
+          echo "Database initialized and configured successfully."
+
+          pg_ctl stop --pgdata="${POSTGRES_DATA}" --silent --wait --timeout=300 > /dev/null 2>&1
+          if [ $? -ne 0 ]; then
+            echo "Failed to stop PostgreSQL."
+            exit 1
+          fi
+
+          exec postgres -D "${POSTGRES_DATA}" -p "${POSTGRES_PORT}"
+          {% endraw %}
+        EOH
+        destination = "local/run.sh"
+        change_mode = "noop"
+        perms = "0755"
+      }
+
+      config {
+        command = "bash"
+        args = ["-c", "exec local/run.sh"]
+      }
+    }
+
     task "preconfrpc" {
       driver = "exec"
 
@@ -90,6 +200,12 @@ job "{{ job.name }}" {
           PRECONF_RPC_BIDDER_RPC_URL="{{ .Address }}:{{ .Port }}"
             {{- end }}
           {{- end }}
+          {{- range nomadService "{% endraw %}{{ job.name }}{% raw %}" }}
+            {{- if contains "db" .Tags }}
+          PRECONF_RPC_PG_HOST="localhost"
+          PRECONF_RPC_PG_PORT="{{ .Port }}"
+            {{- end }}
+          {{- end }}
           {% endraw %}
           XDG_CONFIG_HOME="local/.config"
           {% if profile == 'preconf-rpc-test' %}
@@ -177,6 +293,19 @@ job "{{ job.name }}" {
             echo "Failed to send funds to: ${ADDRESS}"
           fi
           {{- end }}
+
+          source alloc/data/postgres.env
+          export PRECONF_RPC_PG_USER="${POSTGRES_USERNAME}"
+          export PRECONF_RPC_PG_PASSWORD="${POSTGRES_PASSWORD}"
+          export PRECONF_RPC_PG_DBNAME="${POSTGRES_DB}"
+
+          export PRECONF_RPC_DEPOSIT_ADDRESS="$(echo '{{ $secret.Data.data.deposit_keystore }}' | jq -r '.address')"
+          export PRECONF_RPC_BRIDGE_ADDRESS="$(echo '{{ $secret.Data.data.bridge_keystore }}' | jq -r '.address')"
+
+          if ! timeout 5m bash -c 'until pg_isready -h ${PRECONF_RPC_PG_HOST} -p ${PRECONF_RPC_PG_PORT} -U ${PRECONF_RPC_PG_USER} -d ${PRECONF_RPC_PG_DBNAME}; do sleep 2; done'; then
+            echo "Waiting for PostgreSQL to start..."
+            sleep 3
+          fi
           {% endraw %}
 
           chmod +x local/preconf-rpc

infrastructure/nomad/playbooks/variables/profiles.yml

@@ -748,10 +748,16 @@ jobs:
       - *preconf_rpc_artifact
       - keystores:
           preconf_rpc_keystore:
+          deposit_keystore:
+          bridge_keystore:
     count: 1
     target: *mev_commit_bidder_node1_job
     ports:
-      - http:
+      - db:
+          static: 5434
+          to: 5434
+        http:
+          static: 10545
           to: 8080
     env:
       l1_chain_id: "{{ environments[env].chain_id }}"

tools/go.mod

@@ -23,29 +23,49 @@ require (
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2
-	github.com/cockroachdb/pebble v1.1.2
 	github.com/google/go-cmp v0.6.0
-	resenje.org/multex v0.2.0
+	github.com/testcontainers/testcontainers-go v0.27.0
 )
 
 require (
-	github.com/DataDog/zstd v1.5.5 // indirect
-	github.com/cockroachdb/errors v1.11.3 // indirect
-	github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect
-	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
-	github.com/cockroachdb/redact v1.1.5 // indirect
-	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/containerd/containerd v1.7.11 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v25.0.6+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect
-	github.com/getsentry/sentry-go v0.28.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/shirou/gopsutil/v3 v3.23.11 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 )
 
@@ -95,7 +115,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	golang.org/x/crypto v0.35.0
 	golang.org/x/net v0.36.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/sync v0.11.0
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect