config(exesh): override cfg via env

Exesh/cmd/coordinator/main.go

@@ -55,7 +55,7 @@ func main() {
 		return
 	}
 
-	fs, err := filestorage.New(log, cfg.FileStorage, mux)
+	fs, err := filestorage.New(log, cfg.FileStorage.ToExternal(), mux)
 	if err != nil {
 		log.Error("failed to create filestorage", slog.String("error", err.Error()))
 		return

Exesh/cmd/worker/main.go

@@ -45,7 +45,7 @@ func main() {
 
 	mux := chi.NewRouter()
 
-	fs, err := filestorage.New(log, cfg.FileStorage, mux)
+	fs, err := filestorage.New(log, cfg.FileStorage.ToExternal(), mux)
 	if err != nil {
 		log.Error("failed to create filestorage", slog.String("error", err.Error()))
 		return

Exesh/config/worker-1/docker.yml → Exesh/config/worker.yml

@@ -3,7 +3,7 @@ http_server:
   addr: 0.0.0.0:5254
   metrics_addr: 0.0.0.0:9091
 filestorage:
-  root_dir: file_storage/worker-1
+  root_dir: file_storage/worker
   trasher:
     workers: 1
     collector_iterations_delay: 60
@@ -14,7 +14,7 @@ source_provider:
 output_provider:
   artifact_ttl: 5m
 worker:
-  id: http://worker-1:5254
+  id: http://worker:5254
   free_slots: 2
   coordinator_endpoint: http://coordinator:5253
   heartbeat_delay: 100ms

Exesh/docker-compose.yml

@@ -40,7 +40,10 @@ services:
       - "5253:5253"
       - "9080:9090"
     environment:
-      CONFIG_PATH: /config/coordinator/docker.yml
+      CONFIG_PATH: /config/coordinator.yml
+      EXESH_HTTP_SERVER_ADDR: 0.0.0.0:5253
+      EXESH_HTTP_SERVER_METRICS_ADDR: 0.0.0.0:9090
+      EXESH_JOB_FACTORY_FILESTORAGE_ENDPOINT: http://coordinator:5253
     depends_on:
       postgres:
         condition: service_healthy
@@ -60,7 +63,12 @@ services:
       - "5254:5254"
       - "9081:9091"
     environment:
-      CONFIG_PATH: /config/worker-1/docker.yml
+      CONFIG_PATH: /config/worker.yml
+      EXESH_HTTP_SERVER_ADDR: 0.0.0.0:5254
+      EXESH_HTTP_SERVER_METRICS_ADDR: 0.0.0.0:9091
+      EXESH_FILE_STORAGE_ROOT_DIR: file_storage/worker-1
+      EXESH_WORKER_ID: http://worker-1:5254
+      EXESH_WORKER_COORDINATOR_ENDPOINT: http://coordinator:5253
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./config:/config:ro
@@ -78,7 +86,12 @@ services:
       - "5255:5255"
       - "9082:9092"
     environment:
-      CONFIG_PATH: /config/worker-2/docker.yml
+      CONFIG_PATH: /config/worker.yml
+      EXESH_HTTP_SERVER_ADDR: 0.0.0.0:5255
+      EXESH_HTTP_SERVER_METRICS_ADDR: 0.0.0.0:9092
+      EXESH_FILE_STORAGE_ROOT_DIR: file_storage/worker-2
+      EXESH_WORKER_ID: http://worker-2:5255
+      EXESH_WORKER_COORDINATOR_ENDPOINT: http://coordinator:5253
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - ./config:/config:ro

Exesh/internal/config/config.go

@@ -2,7 +2,7 @@ package config
 
 type (
 	HttpServerConfig struct {
-		Addr        string `yaml:"addr"`
-		MetricsAddr string `yaml:"metrics_addr"`
+		Addr        string `yaml:"addr" env:"ADDR"`
+		MetricsAddr string `yaml:"metrics_addr" env:"METRICS_ADDR"`
 	}
 )

Exesh/internal/config/coordinator_config.go

@@ -5,56 +5,55 @@ import (
 	"os"
 	"time"
 
-	filestorage "github.com/DIvanCode/filestorage/pkg/config"
 	"github.com/ilyakaznacheev/cleanenv"
 )
 
 type (
 	CoordinatorConfig struct {
-		Env                string                   `yaml:"env"`
-		HttpServer         HttpServerConfig         `yaml:"http_server"`
-		Storage            StorageConfig            `yaml:"storage"`
-		FileStorage        filestorage.Config       `yaml:"filestorage"`
-		JobFactory         JobFactoryConfig         `yaml:"job_factory"`
-		ExecutionScheduler ExecutionSchedulerConfig `yaml:"execution_scheduler"`
-		WorkerPool         WorkerPoolConfig         `yaml:"worker_pool"`
-		ArtifactRegistry   ArtifactRegistryConfig   `yaml:"artifact_registry"`
-		Sender             SenderConfig             `yaml:"sender"`
+		Env                string                   `yaml:"env" env:"EXESH_ENV"`
+		HttpServer         HttpServerConfig         `yaml:"http_server" env-prefix:"EXESH_HTTP_SERVER_"`
+		Storage            StorageConfig            `yaml:"storage" env-prefix:"EXESH_STORAGE_"`
+		FileStorage        FileStorageConfig        `yaml:"filestorage" env-prefix:"EXESH_FILE_STORAGE_"`
+		JobFactory         JobFactoryConfig         `yaml:"job_factory" env-prefix:"EXESH_JOB_FACTORY_"`
+		ExecutionScheduler ExecutionSchedulerConfig `yaml:"execution_scheduler" env-prefix:"EXESH_EXECUTION_SCHEDULER_"`
+		WorkerPool         WorkerPoolConfig         `yaml:"worker_pool" env-prefix:"EXESH_WORKER_POOL_"`
+		ArtifactRegistry   ArtifactRegistryConfig   `yaml:"artifact_registry" env-prefix:"EXESH_ARTIFACT_REGISTRY_"`
+		Sender             SenderConfig             `yaml:"sender" env-prefix:"EXESH_SENDER_"`
 	}
 
 	StorageConfig struct {
-		ConnectionString string        `yaml:"connection_string"`
-		InitTimeout      time.Duration `yaml:"init_timeout"`
+		ConnectionString string        `yaml:"connection_string" env:"CONNECTION_STRING"`
+		InitTimeout      time.Duration `yaml:"init_timeout" env:"INIT_TIMEOUT"`
 	}
 
 	ExecutionSchedulerConfig struct {
-		ExecutionsInterval  time.Duration `yaml:"executions_interval"`
-		MaxConcurrency      int           `yaml:"max_concurrency"`
-		ExecutionRetryAfter time.Duration `yaml:"execution_retry_after"`
+		ExecutionsInterval  time.Duration `yaml:"executions_interval" env:"EXECUTIONS_INTERVAL"`
+		MaxConcurrency      int           `yaml:"max_concurrency" env:"MAX_CONCURRENCY"`
+		ExecutionRetryAfter time.Duration `yaml:"execution_retry_after" env:"EXECUTION_RETRY_AFTER"`
 	}
 
 	JobFactoryConfig struct {
 		Output struct {
-			CompiledBinary string `yaml:"compiled_binary"`
-			RunOutput      string `yaml:"run_output"`
-		} `yaml:"output"`
+			CompiledBinary string `yaml:"compiled_binary" env:"COMPILED_BINARY"`
+			RunOutput      string `yaml:"run_output" env:"RUN_OUTPUT"`
+		} `yaml:"output" env-prefix:"OUTPUT_"`
 		SourceTTL struct {
-			FilestorageBucket time.Duration `yaml:"filestorage_bucket"`
-		} `yaml:"source_ttl"`
-		FilestorageEndpoint string `yaml:"filestorage_endpoint"`
+			FilestorageBucket time.Duration `yaml:"filestorage_bucket" env:"FILESTORAGE_BUCKET"`
+		} `yaml:"source_ttl" env-prefix:"SOURCE_TTL_"`
+		FilestorageEndpoint string `yaml:"filestorage_endpoint" env:"FILESTORAGE_ENDPOINT"`
 	}
 
 	WorkerPoolConfig struct {
-		WorkerDieAfter time.Duration `yaml:"worker_die_after"`
+		WorkerDieAfter time.Duration `yaml:"worker_die_after" env:"WORKER_DIE_AFTER"`
 	}
 
 	ArtifactRegistryConfig struct {
-		ArtifactTTL time.Duration `yaml:"artifact_ttl"`
+		ArtifactTTL time.Duration `yaml:"artifact_ttl" env:"ARTIFACT_TTL"`
 	}
 
 	SenderConfig struct {
-		Brokers []string `yaml:"brokers"`
-		Topic   string   `yaml:"topic"`
+		Brokers []string `yaml:"brokers" env:"BROKERS" env-separator:","`
+		Topic   string   `yaml:"topic" env:"TOPIC"`
 	}
 )
 

Exesh/internal/config/filestorage_config.go

@@ -0,0 +1,27 @@
+package config
+
+import filestorage "github.com/DIvanCode/filestorage/pkg/config"
+
+type (
+	FileStorageConfig struct {
+		RootDir string                   `yaml:"root_dir" env:"ROOT_DIR"`
+		Trasher FileStorageTrasherConfig `yaml:"trasher" env-prefix:"TRASHER_"`
+	}
+
+	FileStorageTrasherConfig struct {
+		Workers                  int `yaml:"workers" env:"WORKERS"`
+		CollectorIterationsDelay int `yaml:"collector_iterations_delay" env:"COLLECTOR_ITERATIONS_DELAY"`
+		WorkerIterationsDelay    int `yaml:"worker_iterations_delay" env:"WORKER_ITERATIONS_DELAY"`
+	}
+)
+
+func (c FileStorageConfig) ToExternal() filestorage.Config {
+	return filestorage.Config{
+		RootDir: c.RootDir,
+		Trasher: filestorage.TrasherConfig{
+			Workers:                  c.Trasher.Workers,
+			CollectorIterationsDelay: c.Trasher.CollectorIterationsDelay,
+			WorkerIterationsDelay:    c.Trasher.WorkerIterationsDelay,
+		},
+	}
+}

Exesh/internal/config/worker_config.go

@@ -5,35 +5,34 @@ import (
 	"os"
 	"time"
 
-	filestorage "github.com/DIvanCode/filestorage/pkg/config"
 	"github.com/ilyakaznacheev/cleanenv"
 )
 
 type (
 	WorkerConfig struct {
-		Env            string               `yaml:"env"`
-		HttpServer     HttpServerConfig     `yaml:"http_server"`
-		FileStorage    filestorage.Config   `yaml:"filestorage"`
-		SourceProvider SourceProviderConfig `yaml:"input_provider"`
-		OutputProvider OutputProviderConfig `yaml:"output_provider"`
-		Worker         WorkConfig           `yaml:"worker"`
+		Env            string               `yaml:"env" env:"EXESH_ENV"`
+		HttpServer     HttpServerConfig     `yaml:"http_server" env-prefix:"EXESH_HTTP_SERVER_"`
+		FileStorage    FileStorageConfig    `yaml:"filestorage" env-prefix:"EXESH_FILE_STORAGE_"`
+		SourceProvider SourceProviderConfig `yaml:"source_provider" env-prefix:"EXESH_SOURCE_PROVIDER_"`
+		OutputProvider OutputProviderConfig `yaml:"output_provider" env-prefix:"EXESH_OUTPUT_PROVIDER_"`
+		Worker         WorkConfig           `yaml:"worker" env-prefix:"EXESH_WORKER_"`
 	}
 
 	SourceProviderConfig struct {
-		FilestorageBucketTTL time.Duration `yaml:"filestorage_bucket_ttl"`
-		ArtifactTTL          time.Duration `yaml:"artifact_ttl"`
+		FilestorageBucketTTL time.Duration `yaml:"filestorage_bucket_ttl" env:"FILESTORAGE_BUCKET_TTL"`
+		ArtifactTTL          time.Duration `yaml:"artifact_ttl" env:"ARTIFACT_TTL"`
 	}
 
 	OutputProviderConfig struct {
-		ArtifactTTL time.Duration `yaml:"artifact_ttl"`
+		ArtifactTTL time.Duration `yaml:"artifact_ttl" env:"ARTIFACT_TTL"`
 	}
 
 	WorkConfig struct {
-		WorkerID            string        `yaml:"id"`
-		FreeSlots           int           `yaml:"free_slots"`
-		CoordinatorEndpoint string        `yaml:"coordinator_endpoint"`
-		HeartbeatDelay      time.Duration `yaml:"heartbeat_delay"`
-		WorkerDelay         time.Duration `yaml:"worker_delay"`
+		WorkerID            string        `yaml:"id" env:"ID"`
+		FreeSlots           int           `yaml:"free_slots" env:"FREE_SLOTS"`
+		CoordinatorEndpoint string        `yaml:"coordinator_endpoint" env:"COORDINATOR_ENDPOINT"`
+		HeartbeatDelay      time.Duration `yaml:"heartbeat_delay" env:"HEARTBEAT_DELAY"`
+		WorkerDelay         time.Duration `yaml:"worker_delay" env:"WORKER_DELAY"`
 	}
 )
 

Exesh/internal/worker/worker.go

@@ -82,7 +82,6 @@ func (w *Worker) runHeartbeat(ctx context.Context) {
 		}
 
 		if w.getFreeSlots() == 0 {
-			w.log.Debug("skip heartbeat loop (no free slots)")
 			continue
 		}