From 9ed547ef85e1f47d7a839024a847356e46ec9393 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:15:03 +0200 Subject: [PATCH 1/9] docker-compose replaced 'bitnami/mysql' with 'mysql' due to errors --- api/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 1026cf8..8e93827 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -2,7 +2,7 @@ #DON'T USE IT IN PRODUCTION IT CONTAINS HARDCODED PASSWORDS services: mysql: - image: 'bitnami/mysql:8.3.0' + image: 'mysql:8.4.0' environment: - MYSQL_ROOT_USER=root - MYSQL_ROOT_PASSWORD=Root#123 From f101ddedf4828c5f374ac887ee7ebabfa62046b0 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:19:52 +0200 Subject: [PATCH 2/9] load environment variables instead of config files replaced config(*).yml with .env(*) --- api/.dockerignore | 3 ++- api/.env | 10 ++++++++++ api/.env.deployment | 9 +++++++++ api/config.deployment.yml | 9 --------- api/config.yml | 9 --------- api/docker-compose.yml | 11 ++++------- 6 files changed, 25 insertions(+), 26 deletions(-) create mode 100644 api/.env create mode 100644 api/.env.deployment delete mode 100644 api/config.deployment.yml delete mode 100644 api/config.yml diff --git a/api/.dockerignore b/api/.dockerignore index 7513741..2f9c94d 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -1,4 +1,5 @@ -config.yml +.env +.env.deployment README.md .idea internal diff --git a/api/.env b/api/.env new file mode 100644 index 0000000..d5e69e1 --- /dev/null +++ b/api/.env @@ -0,0 +1,10 @@ +PORT=8080 +GIN_MODE="debug"#["release", "debug"] + +MYSQL_DATABASE="api"#database name +MYSQL_ROOT_USER="root"#login name +MYSQL_ROOT_PASSWORD="Root#123" +MYSQL_HOST="localhost" +MYSQL_PORT=3306 + + diff --git a/api/.env.deployment b/api/.env.deployment new file mode 100644 index 0000000..614a6a8 --- /dev/null +++ b/api/.env.deployment @@ -0,0 +1,9 @@ +PORT="8080" +GIN_MODE="release"#["release", "debug"] + +MYSQL_DATABASE="api"#database name +MYSQL_ROOT_USER="root"#login name +MYSQL_ROOT_PASSWORD="changeme" #TODO change me. +MYSQL_HOST="mysql" +MYSQL_PORT="3306" + diff --git a/api/config.deployment.yml b/api/config.deployment.yml deleted file mode 100644 index b91c431..0000000 --- a/api/config.deployment.yml +++ /dev/null @@ -1,9 +0,0 @@ -## PRODUCTIVE CONFIG -PORT: 8080 -GIN_MODE: release #["release", "debug"] -DATABASE: - NAME: api #database name - USER: root #This login name is just for local deployment. TODO set it in productive deployment - PASSWORD: Root#123 #This password is just for local deployment. TODO set it in productive deployment - HOST: mysql - PORT: 3306 diff --git a/api/config.yml b/api/config.yml deleted file mode 100644 index cc996b9..0000000 --- a/api/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -PORT: 8080 -GIN_MODE: debug #["release", "debug"] -DATABASE: - NAME: api #database name - USER: root #login name - PASSWORD: Root#123 #This password is of course not production. Just local deployment - HOST: localhost - PORT: 3306 - diff --git a/api/docker-compose.yml b/api/docker-compose.yml index 8e93827..e6111ac 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -1,14 +1,10 @@ #The docker-compose is for local deployment. -#DON'T USE IT IN PRODUCTION IT CONTAINS HARDCODED PASSWORDS services: mysql: + env_file: .env.deployment image: 'mysql:8.4.0' - environment: - - MYSQL_ROOT_USER=root - - MYSQL_ROOT_PASSWORD=Root#123 - - MYSQL_DATABASE=api ports: - - "3306:3306" + - "${MYSQL_PORT}:${MYSQL_PORT}" volumes: - ./internal/database:/bitnami/mysql/data networks: @@ -24,9 +20,10 @@ services: # - "--socket=/opt/bitnami/mysql/tmp/mysql.sock" # - "--pid-file=/opt/bitnami/mysql/tmp/mysqld.pid" api: + env_file: .env.deployment build: . ports: - - "8080:8080" + - "${PORT}:${PORT}" restart: on-failure networks: - api From 44e5f786eb354318e3842425e9477fca3d0a70bd Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:20:21 +0200 Subject: [PATCH 3/9] Dockerfile doesn't copy config file anymore because we have it in docker-compose --- api/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 774b58c..61d431d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -18,8 +18,6 @@ CMD ["go", "test", "./tests"] #Stage 2b: Run Api FROM scratch as release -#Copy config -COPY config.deployment.yml ./config.yml #Copy migration scripts COPY migrations migrations #Copy build result to next stage From d1f79a221f026ae7035f68c7c64999f754c0337d Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:20:42 +0200 Subject: [PATCH 4/9] updated healthcheck in docker-compose use mysqladmin ping --- api/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/docker-compose.yml b/api/docker-compose.yml index e6111ac..fbfd84c 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -10,7 +10,7 @@ services: networks: - api healthcheck: - test: mysql --user=root --password=Root#123 -e 'SELECT * FROM mysql.time_zone' + test: mysqladmin ping --host=${MYSQL_HOST} --port=${MYSQL_PORT} --user=${MYSQL_ROOT_USER} --password=${MYSQL_ROOT_PASSWORD} restart: on-failure #command: # fix from https://github.com/bitnami/containers/issues/44854#issuecomment-1800945882 # - "/opt/bitnami/mysql/bin/mysqld" From a79dbe224630da2e55bd6d877077e508d9ffe704 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:21:32 +0200 Subject: [PATCH 5/9] golang script changes due to environment variables replaced library; read environment variables --- api/cmd/main.go | 18 +++++++++++------- api/go.mod | 2 +- api/go.sum | 2 ++ api/scripts/databaseScripts.go | 21 ++++++++++----------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/api/cmd/main.go b/api/cmd/main.go index f5b153e..ca09e0d 100644 --- a/api/cmd/main.go +++ b/api/cmd/main.go @@ -9,9 +9,10 @@ import ( "fmt" "github.com/gin-gonic/gin" _ "github.com/go-sql-driver/mysql" - "github.com/spf13/viper" + "github.com/joho/godotenv" "log" "net/http" + "os" ) func setupRouter(db *sql.DB) *gin.Engine { @@ -41,17 +42,17 @@ func setupRouter(db *sql.DB) *gin.Engine { } func loadConfig() { - viper.SetConfigFile("config.yml") - err := viper.ReadInConfig() + err := godotenv.Load(".env") if err != nil { - log.Fatal(err.Error()) + log.Println(fmt.Sprintf("Failed to load .env file: %s", err)) + log.Println("Environment variables will be used") } } func setupDatabase() *sql.DB { //Create database if it is not existing yet. //We might have to remove this if we use an azure database - scripts.CreateDatabaseIfNotExists(viper.GetString("DATABASE.NAME")) + scripts.CreateDatabaseIfNotExists(os.Getenv("MYSQL_DATABASE")) //Connect to the database db := scripts.ConnectToDatabase() //Check if database is online @@ -73,11 +74,14 @@ func main() { defer db.Close() //Set Gin-gonic to debug or release mode - gin.SetMode(viper.GetString("GIN_MODE")) + gin.SetMode(os.Getenv("GIN_MODE")) //Setup Routes r := setupRouter(db) // Listen and Server in 0.0.0.0:8080 - r.Run(fmt.Sprintf(":%d", viper.GetInt("port"))) + err := r.Run(fmt.Sprintf(":%s", os.Getenv("PORT"))) + if err != nil { + log.Fatal(err.Error()) + } } diff --git a/api/go.mod b/api/go.mod index 36cd187..ff57214 100644 --- a/api/go.mod +++ b/api/go.mod @@ -8,7 +8,6 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-sql-driver/mysql v1.8.1 github.com/google/uuid v1.4.0 - github.com/spf13/viper v1.18.2 ) require ( @@ -24,6 +23,7 @@ require ( github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect diff --git a/api/go.sum b/api/go.sum index d326bfb..7762e57 100644 --- a/api/go.sum +++ b/api/go.sum @@ -46,6 +46,8 @@ github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= diff --git a/api/scripts/databaseScripts.go b/api/scripts/databaseScripts.go index 4050b15..1d318c7 100644 --- a/api/scripts/databaseScripts.go +++ b/api/scripts/databaseScripts.go @@ -4,7 +4,6 @@ import ( "api/shared" "database/sql" "fmt" - "github.com/spf13/viper" "log" "os" "regexp" @@ -16,11 +15,11 @@ import ( func ConnectToDatabase() *sql.DB { // connect to db using standard Go database/sql API connectionString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", - viper.GetString("DATABASE.USER"), - viper.GetString("DATABASE.PASSWORD"), - viper.GetString("DATABASE.HOST"), - viper.GetString("DATABASE.PORT"), - viper.GetString("DATABASE.NAME")) + os.Getenv("MYSQL_ROOT_USER"), + os.Getenv("MYSQL_ROOT_PASSWORD"), + os.Getenv("MYSQL_HOST"), + os.Getenv("MYSQL_PORT"), + os.Getenv("MYSQL_DATABASE")) db, err := sql.Open("mysql", connectionString) @@ -102,7 +101,7 @@ func MigrateDatabase(db *sql.DB) { log.Println("Finished migrations") } -// getMigrationIds returns the Ids of migrations which have been applied to the database. +// getMigrationIds returns the Ids of migrations which have been applied to the database func getMigrationIds(db *sql.DB) []int { var migrations []int @@ -133,10 +132,10 @@ func getMigrationIds(db *sql.DB) []int { func CreateDatabaseIfNotExists(database string) { // connect to db using standard Go database/sql API connectionString := fmt.Sprintf("%s:%s@tcp(%s:%s)/", - viper.GetString("DATABASE.USER"), - viper.GetString("DATABASE.PASSWORD"), - viper.GetString("DATABASE.HOST"), - viper.GetString("DATABASE.PORT")) + os.Getenv("MYSQL_ROOT_USER"), + os.Getenv("MYSQL_ROOT_PASSWORD"), + os.Getenv("MYSQL_HOST"), + os.Getenv("MYSQL_PORT")) db, err := sql.Open("mysql", connectionString) From 37e1c593cb4671541af671a8bdebfc6635de0e87 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:21:46 +0200 Subject: [PATCH 6/9] readme added info to change mysql password --- api/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/README.md b/api/README.md index f6ec256..551a076 100644 --- a/api/README.md +++ b/api/README.md @@ -1,4 +1,5 @@ ## TL;DR -`` docker compose build; `` `` docker compose up `` - +Change MYSQL password in .env.deployment +`` docker compose build; `` `` docker compose up -d`` The api will be exposed to port 8080, access it with `localhost:8080`. + From a8bee31c0a7fa7560afb24740b46bfda82de853a Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:22:59 +0200 Subject: [PATCH 7/9] docker-compose remove command comment --- api/docker-compose.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/api/docker-compose.yml b/api/docker-compose.yml index fbfd84c..9237152 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -12,13 +12,6 @@ services: healthcheck: test: mysqladmin ping --host=${MYSQL_HOST} --port=${MYSQL_PORT} --user=${MYSQL_ROOT_USER} --password=${MYSQL_ROOT_PASSWORD} restart: on-failure - #command: # fix from https://github.com/bitnami/containers/issues/44854#issuecomment-1800945882 - # - "/opt/bitnami/mysql/bin/mysqld" - # - "--defaults-file=/opt/bitnami/mysql/conf/my.cnf" - # - "--basedir=/opt/bitnami/mysql" - # - "--datadir=/bitnami/mysql/data" - # - "--socket=/opt/bitnami/mysql/tmp/mysql.sock" - # - "--pid-file=/opt/bitnami/mysql/tmp/mysqld.pid" api: env_file: .env.deployment build: . From 9dc8e1e540546e25f699a7019a65725de77740df Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 18:45:48 +0200 Subject: [PATCH 8/9] added environment variables to readme --- api/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/README.md b/api/README.md index 551a076..fc048cc 100644 --- a/api/README.md +++ b/api/README.md @@ -3,3 +3,18 @@ Change MYSQL password in .env.deployment `` docker compose build; `` `` docker compose up -d`` The api will be exposed to port 8080, access it with `localhost:8080`. + +## Environment variables +The docker image will use the following environment variables: + +| Key | Default Value | Options | +|----------------------------------------------------|---------------|--------------------| +| PORT | "8080" | | +| GIN_MODE | "release" | "release", "debug" | +| MYSQL_HOST | "mysql" | | +| MYSQL_PORT | "3306" | | +| MYSQL_DATABASE | "api" | | +| MYSQL_ROOT_USER | "root" | | +| MYSQL_ROOT_PASSWORD | "changeme" | | + +If you use the docker image directly (without our provided docker-compose), you must specify them. \ No newline at end of file From 78d3d4d95d52879cef46d11d57ee777850374150 Mon Sep 17 00:00:00 2001 From: Kai Date: Mon, 27 May 2024 19:07:57 +0200 Subject: [PATCH 9/9] added info message when environment variables are not set --- api/scripts/databaseScripts.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/scripts/databaseScripts.go b/api/scripts/databaseScripts.go index 1d318c7..d0b0379 100644 --- a/api/scripts/databaseScripts.go +++ b/api/scripts/databaseScripts.go @@ -140,13 +140,16 @@ func CreateDatabaseIfNotExists(database string) { db, err := sql.Open("mysql", connectionString) if err != nil { - log.Fatal(err) + log.Fatal("Error connecting to database: ", err) } defer db.Close() _, err = db.Exec(fmt.Sprintf("Create database if not exists %s", database)) if err != nil { - log.Fatal(err) + if connectionString == ":@tcp(:)/" { + log.Println("Environment variables have not been set. See https://github.com/AustrianDataLAB/IndieGameStream/tree/develop/api") + } + log.Fatal("Error creating database: ", err) } return