diff --git a/Makefile b/Makefile index 30c45df397..45d94a4172 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ # make dev.attach.credentials # make dev.pull.registrar+studio # make dev.up.lms -# make dev.up.without-deps.lms+forum+discovery+mysql57+elasticsearch+memcached -# make dev.restart-container.mysql57+lms +# make dev.up.without-deps.lms+forum+discovery+mysql80+elasticsearch+memcached +# make dev.restart-container.mysql80+lms # There are also "prefix-form" targets, which are simply an alternate way to spell # the 'dev.' targets. @@ -230,8 +230,8 @@ impl-dev.provision.%: dev.check-memory ## Provision specified services. dev.provision.%: ## Provision specified services. @scripts/send_metrics.py wrap "dev.provision.$*" -dev.backup: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host. - docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql57.tar.gz /var/lib/mysql +dev.backup: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Write all data volumes to the host. + docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mysql80.tar.gz /var/lib/mysql docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/mongo.tar.gz /data/db docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch.tar.gz /usr/share/elasticsearch/data docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/elasticsearch7.tar.gz /usr/share/elasticsearch/data @@ -239,8 +239,8 @@ dev.backup: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+o docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.opensearch12) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/opensearch12.tar.gz /usr/share/opensearch/data docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.coursegraph) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zcvf /backup/coursegraph.tar.gz /data -dev.restore: dev.up.mysql57+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA! - docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql57) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql57.tar.gz +dev.restore: dev.up.mysql80+mongo+elasticsearch+elasticsearch7+elasticsearch710+opensearch12+coursegraph ## Restore all data volumes from the host. WILL OVERWRITE ALL EXISTING DATA! + docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mysql80) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mysql80.tar.gz docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.mongo) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/mongo.tar.gz docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch.tar.gz docker run --rm --volumes-from $$(make --silent --no-print-directory dev.print-container.elasticsearch7) -v $$(pwd)/.dev/backups:/backup debian:jessie tar zxvf /backup/elasticsearch7.tar.gz @@ -270,7 +270,7 @@ dev.migrate.%: ## Run migrations on a service. dev.drop-db: _expects-database.dev.drop-db dev.drop-db.%: ## Irreversably drop the contents of a MySQL database in each mysql container. - docker-compose exec -T mysql57 bash -c "mysql --execute=\"DROP DATABASE $*;\"" + docker-compose exec -T mysql80 bash -c "mysql --execute=\"DROP DATABASE $*;\"" ######################################################################################## @@ -457,10 +457,10 @@ dev.shell.%: ## Run a shell on the specified service's container. docker-compose exec $* /bin/bash dev.dbshell: - docker-compose exec mysql57 bash -c "mysql" + docker-compose exec mysql80 bash -c "mysql" dev.dbshell.%: ## Run a SQL shell on the given database. - docker-compose exec mysql57 bash -c "mysql $*" + docker-compose exec mysql80 bash -c "mysql $*" # List of Makefile targets to run static asset generation, in the form dev.static.$(service) # Services will only have their asset generation added here diff --git a/README.rst b/README.rst index f6fd7565d9..6e5c05402c 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ You can also browse all the documentation in `Read the Docs`_. Notices ------- -**NOTE:** LMS is now using MySql 5.7 by default. You have to run ``make dev.pull.lms`` and ``make dev.provision.lms`` (more details in `Getting Started`_) to fetch latest images and reprovision local copies of databases in order for an existing devstack setup to keep working. +**NOTE:** LMS is now using MySql 8.0 by default. You have to run ``make dev.pull.lms`` and ``make dev.provision.lms`` (more details in `Getting Started`_) to fetch latest images and reprovision local copies of databases in order for an existing devstack setup to keep working. Getting Started diff --git a/docker-compose.yml b/docker-compose.yml index 5831f978eb..5da4211ac5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -237,22 +237,23 @@ services: volumes: - mongo_data:/data/db - mysql57: + mysql80: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci - container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql57" - hostname: mysql57.devstack.edx + container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.mysql80" + hostname: mysql80.devstack.edx environment: MYSQL_ROOT_PASSWORD: "" MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - image: mysql:5.7 + image: mysql:8.0.28-oracle networks: default: aliases: + - edx.devstack.mysql80 - edx.devstack.mysql57 ports: - "3506:3306" volumes: - - mysql57_data:/var/lib/mysql + - mysql80_data:/var/lib/mysql redis: container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.redis" @@ -309,13 +310,13 @@ services: depends_on: - lms - memcached - - mysql57 + - mysql80 # Allows attachment to the credentials service using 'docker attach '. stdin_open: true tty: true environment: CACHE_LOCATION: edx.devstack.memcached:11211 - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 SOCIAL_AUTH_EDX_OIDC_URL_ROOT: http://edx.devstack.lms:18000/oauth2 ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 @@ -334,14 +335,14 @@ services: depends_on: - elasticsearch710 - memcached - - mysql57 + - mysql80 # Allows attachment to the discovery service using 'docker attach '. stdin_open: true tty: true environment: # This next DB_MIGRATION_HOST line can be removed once edx/configuration has been updated with this value for # a while and most people have had a chance to do a "make pull" to get the latest images. - DB_MIGRATION_HOST: edx.devstack.mysql57 + DB_MIGRATION_HOST: edx.devstack.mysql80 TEST_ELASTICSEARCH_URL: "edx.devstack.elasticsearch710" ENABLE_DJANGO_TOOLBAR: 1 DJANGO_WATCHMAN_TIMEOUT: 30 @@ -363,7 +364,7 @@ services: - discovery - lms - memcached - - mysql57 + - mysql80 # Allows attachment to the ecommerce service using 'docker attach '. stdin_open: true tty: true @@ -387,7 +388,7 @@ services: - devpi - elasticsearch710 - lms - - mysql57 + - mysql80 image: edxops/notes:${OPENEDX_RELEASE:-latest} networks: default: @@ -397,7 +398,7 @@ services: - "18120:18120" environment: DB_ENGINE: "django.db.backends.mysql" - DB_HOST: "edx.devstack.mysql57" + DB_HOST: "edx.devstack.mysql80" DB_NAME: "notes" DB_PASSWORD: "password" DB_PORT: "3306" @@ -437,7 +438,7 @@ services: - forum - memcached - mongo - - mysql57 + - mysql80 - opensearch12 # Allows attachment to the LMS service using 'docker attach '. stdin_open: true @@ -469,14 +470,14 @@ services: hostname: insights.devstack.edx depends_on: - analyticsapi - - mysql57 + - mysql80 - lms - memcached # Allows attachment to the insights service using 'docker attach '. stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: dashboard DB_PORT: 3306 DB_USER: analytics001 @@ -498,13 +499,13 @@ services: container_name: edx.devstack.analyticsapi hostname: analyticsapi depends_on: - - mysql57 + - mysql80 - elasticsearch710 command: bash -c 'source /edx/app/analytics_api/analytics_api_env && while true; do python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:19001 --settings analyticsdataserver.settings.devstack; sleep 2; done' stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_PORT: 3306 DB_USER: analytics001 DB_PASSWORD: password @@ -521,7 +522,7 @@ services: depends_on: - discovery - lms - - mysql57 + - mysql80 - memcached - redis - registrar-worker @@ -529,7 +530,7 @@ services: stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: registrar DB_PORT: 3306 DB_USER: registrar001 @@ -559,12 +560,12 @@ services: hostname: registrar-worker.devstack.edx depends_on: - lms - - mysql57 + - mysql80 - redis stdin_open: true tty: true environment: - DB_HOST: edx.devstack.mysql57 + DB_HOST: edx.devstack.mysql80 DB_NAME: registrar DB_PORT: 3306 DB_USER: registrar001 @@ -597,7 +598,7 @@ services: - lms - memcached - mongo - - mysql57 + - mysql80 - opensearch12 # Allows attachment to the Studio service using 'docker attach '. stdin_open: true @@ -630,7 +631,7 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue depends_on: - - mysql57 + - mysql80 networks: default: aliases: @@ -645,7 +646,7 @@ services: volumes: - ${DEVSTACK_WORKSPACE}/xqueue:/edx/app/xqueue/xqueue depends_on: - - mysql57 + - mysql80 networks: default: aliases: @@ -837,4 +838,4 @@ volumes: elasticsearch710_data: mongo_data: opensearch12_data: - mysql57_data: + mysql80_data: diff --git a/docs/developing_on_named_release_branches.rst b/docs/developing_on_named_release_branches.rst index 1931c5c3b9..8deb91f63e 100644 --- a/docs/developing_on_named_release_branches.rst +++ b/docs/developing_on_named_release_branches.rst @@ -47,7 +47,7 @@ Switch between your Devstack releases by doing the following: Examples of Docker Service Names After Setting the ``COMPOSE_PROJECT_NAME`` variable. Notice that the **devstack-juniper.master** name represents the ``COMPOSE_PROJECT_NAME``. - edx.devstack-juniper.master.lms -- edx.devstack-juniper.master.mysql57 +- edx.devstack-juniper.master.mysql80 Each instance has an isolated set of databases. This could, for example, be used to quickly switch between versions of Open edX without hitting as many issues with migrations, data integrity, etc. diff --git a/docs/devstack_faq.rst b/docs/devstack_faq.rst index 5a0618171c..833988a2d7 100644 --- a/docs/devstack_faq.rst +++ b/docs/devstack_faq.rst @@ -134,7 +134,7 @@ How do I connect to the databases from an outside editor? --------------------------------------------------------- To connect to the databases from an outside editor (such as MySQLWorkbench), -first uncomment these lines from ``docker-compose.yml``'s ``mysql57`` section +first uncomment these lines from ``docker-compose.yml``'s ``mysql80`` section .. code:: yaml @@ -145,8 +145,8 @@ Then bring your mysql container down and back up by running: .. code:: sh - docker-compose stop mysql57 - docker-compose up -d mysql57 + docker-compose stop mysql80 + docker-compose up -d mysql80 Then connect using the values below. Note that the username and password will vary depending on the database. For all of the options, see ``provision.sql``. @@ -158,7 +158,7 @@ vary depending on the database. For all of the options, see ``provision.sql``. If you have trouble connecting, ensure the port was mapped successfully by running ``make dev.ps`` and looking for a line like this: -``edx.devstack.mysql57 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. +``edx.devstack.mysql80 docker-entrypoint.sh mysql ... Up 0.0.0.0:3506→3306/tcp``. How do I build the service images myself? ----------------------------------------- @@ -209,7 +209,7 @@ To access the MySQL shell for a particular database, run: .. code:: sh - make dev.shell.mysql57 + make dev.shell.mysql80 mysql use ; diff --git a/load-db.sh b/load-db.sh index 99be594989..3863639ef1 100755 --- a/load-db.sh +++ b/load-db.sh @@ -16,6 +16,6 @@ then fi echo "Loading the $1 database..." -mysql_container=$(make --silent --no-print-directory dev.print-container.mysql57) +mysql_container=$(make --silent --no-print-directory dev.print-container.mysql80) docker exec -i "$mysql_container" mysql -uroot $1 < $1.sql echo "Finished loading the $1 database!" diff --git a/options.mk b/options.mk index dc2b453001..f56acb8e7a 100644 --- a/options.mk +++ b/options.mk @@ -90,4 +90,4 @@ credentials+discovery+ecommerce+insights+lms+registrar+studio # All third-party services. # Separated by plus signs. Listed in alphabetical order for clarity. THIRD_PARTY_SERVICES ?= \ -chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql57+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica +chrome+coursegraph+devpi+elasticsearch+elasticsearch7+elasticsearch710+firefox+memcached+mongo+mysql80+opensearch12+redis+namenode+datanode+resourcemanager+nodemanager+sparkmaster+sparkworker+vertica diff --git a/provision.sh b/provision.sh index 86932ca331..4d4f3535d2 100755 --- a/provision.sh +++ b/provision.sh @@ -123,14 +123,14 @@ fi echo -e "${GREEN}Will provision the following:\n ${to_provision_ordered}${NC}" # Bring the databases online. -docker-compose up -d mysql57 +docker-compose up -d mysql80 if needs_mongo "$to_provision_ordered"; then docker-compose up -d mongo fi # Ensure the MySQL server is online and usable -echo "${GREEN}Waiting for MySQL 5.7.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +echo "${GREEN}Waiting for MySQL 8.0.${NC}" +until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -140,8 +140,8 @@ done # See https://github.com/docker-library/mysql/issues/245 for why this is necessary. sleep 10 -echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}" -until docker-compose exec -T mysql57 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null +echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}" +until docker-compose exec -T mysql80 bash -e -c "mysql -uroot -se \"SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'root')\"" &> /dev/null do printf "." sleep 1 @@ -152,8 +152,8 @@ echo -e "${GREEN}MySQL ready.${NC}" # Ensure that the MySQL databases and users are created for all IDAs. # (A no-op for databases and users that already exist). -echo -e "${GREEN}Ensuring MySQL 5.7 databases and users exist...${NC}" -docker-compose exec -T mysql57 bash -e -c "mysql -uroot mysql" < provision.sql +echo -e "${GREEN}Ensuring MySQL 8.0 databases and users exist...${NC}" +docker-compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision.sql # If necessary, ensure the MongoDB server is online and usable # and create its users. diff --git a/provision.sql b/provision.sql index 0b672c9986..bf7a0eb2ee 100644 --- a/provision.sql +++ b/provision.sql @@ -1,37 +1,47 @@ CREATE DATABASE IF NOT EXISTS credentials; -GRANT ALL ON credentials.* TO 'credentials001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'credentials001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON credentials.* TO 'credentials001'@'%'; + CREATE DATABASE IF NOT EXISTS discovery; -GRANT ALL ON discovery.* TO 'discov001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'discov001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON discovery.* TO 'discov001'@'%'; CREATE DATABASE IF NOT EXISTS ecommerce; -GRANT ALL ON ecommerce.* TO 'ecomm001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'ecomm001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON ecommerce.* TO 'ecomm001'@'%'; + CREATE DATABASE IF NOT EXISTS notes; -GRANT ALL ON notes.* TO 'notes001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'notes001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON notes.* TO 'notes001'@'%'; CREATE DATABASE IF NOT EXISTS registrar; -GRANT ALL ON registrar.* TO 'registrar001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'registrar001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON registrar.* TO 'registrar001'@'%'; CREATE DATABASE IF NOT EXISTS xqueue; -GRANT ALL ON xqueue.* TO 'xqueue001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'xqueue001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON xqueue.* TO 'xqueue001'@'%'; CREATE DATABASE IF NOT EXISTS edxapp; CREATE DATABASE IF NOT EXISTS edxapp_csmh; -GRANT ALL ON edxapp.* TO 'edxapp001'@'%' IDENTIFIED BY 'password'; -GRANT ALL ON edxapp_csmh.* TO 'edxapp001'@'%'; +CREATE USER IF NOT EXISTS 'edxapp001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'edxapp_csmh'@'%'; +GRANT ALL PRIVILEGES ON edxapp.* TO 'edxapp001'@'%'; +GRANT ALL PRIVILEGES ON edxapp_csmh.* TO 'edxapp001'@'%'; CREATE DATABASE IF NOT EXISTS `dashboard`; -GRANT ALL ON `dashboard`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON dashboard.* TO 'analytics001'@'%'; CREATE DATABASE IF NOT EXISTS `analytics-api`; -GRANT ALL ON `analytics-api`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON `analytics-api`.* TO 'analytics001'@'%'; CREATE DATABASE IF NOT EXISTS `reports`; -GRANT ALL ON `reports`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON reports.* TO 'analytics001'@'%'; CREATE DATABASE IF NOT EXISTS `reports_v1`; -GRANT ALL ON `reports_v1`.* TO 'analytics001'@'%' IDENTIFIED BY 'password'; - +GRANT ALL PRIVILEGES ON `reports_v1`.* TO 'analytics001'@'%'; FLUSH PRIVILEGES; diff --git a/update-dbs-init-sql-scripts.sh b/update-dbs-init-sql-scripts.sh index 91c7de3d84..75b65299f0 100755 --- a/update-dbs-init-sql-scripts.sh +++ b/update-dbs-init-sql-scripts.sh @@ -20,7 +20,7 @@ make dev.clone.ssh make dev.provision.services.lms+ecommerce # dump schema and data from mysql databases in the mysql docker container and copy them to current directory in docker host -MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql57)" +MYSQL_DOCKER_CONTAINER="$(make --silent --no-print-directory dev.print-container.mysql80)" for DB_NAME in "${DBS[@]}"; do DB_CREATION_SQL_SCRIPT="${DB_NAME}.sql" if [[ " ${EDXAPP_DBS[@]} " =~ " ${DB_NAME} " ]]; then