From b6184af2d93c470a382a8005c9fc212d1b0ecccf Mon Sep 17 00:00:00 2001 From: Karol Kulik Date: Mon, 26 Nov 2018 12:03:07 +0100 Subject: [PATCH 1/9] Added flyway support. --- pom.xml | 169 ++++++++++-------- .../flyway/StartupFlywayMigrator.java | 28 +++ .../migration/V1__flyway_schema_baseline.sql | 0 3 files changed, 119 insertions(+), 78 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/flyway/StartupFlywayMigrator.java create mode 100644 src/main/resources/db/migration/V1__flyway_schema_baseline.sql diff --git a/pom.xml b/pom.xml index f91ea6b90ba..fae452e10c9 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 edu.harvard.iq @@ -11,8 +12,8 @@ ${project.build.directory}/endorsed - UTF-8 - -Xdoclint:none + UTF-8 + -Xdoclint:none 1.11.172 @@ -23,8 +24,10 @@ Moving this from plugin config to global config, because of GH-5122. This seems to play well with NetBeans 8.2, IDEA 2018.1 and mvn including compatibility with JaCoCo. --> - -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - + -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} + -Duser.language=${project.language} -Duser.region=${project.region} + + 4.12 5.3.1 5.3.1 @@ -63,10 +66,10 @@ file://${project.basedir}/local_lib - + + If we update glassfish, we should return to the normal sdk as well. --> org.passay @@ -133,10 +136,10 @@ 1.1-SNAPSHOT war - - xerces - xercesImpl - + + xerces + xercesImpl + @@ -193,6 +196,11 @@ jbcrypt 0.3m + + org.flywaydb + flyway-core + 5.2.1 + com.google.guava guava @@ -412,7 +420,7 @@ commons-codec 1.9 - + org.javaswift joss @@ -444,27 +452,27 @@ scribejava-apis 3.1.0 - - - - - - - - com.lyncode - xoai-common - 4.1.0-header-patch - - - com.lyncode - xoai-data-provider - 4.1.0-header-patch - - - com.lyncode - xoai-service-provider - 4.1.0-header-patch - + + + + + + + + com.lyncode + xoai-common + 4.1.0-header-patch + + + com.lyncode + xoai-data-provider + 4.1.0-header-patch + + + com.lyncode + xoai-service-provider + 4.1.0-header-patch + com.google.auto.service @@ -475,16 +483,16 @@ - org.glassfish.jersey.containers - jersey-container-servlet - 2.23.2 - + org.glassfish.jersey.containers + jersey-container-servlet + 2.23.2 + - - org.glassfish.jersey.media - jersey-media-multipart - 2.23.2 - + + org.glassfish.jersey.media + jersey-media-multipart + 2.23.2 + com.mashape.unirest unirest-java @@ -499,22 +507,22 @@ - - + + src/main/java @@ -527,6 +535,7 @@ src/main/resources + **/*.sql **/*.xml @@ -562,9 +571,9 @@ org.apache.maven.plugins maven-war-plugin - 2.3 + 3.2.2 - true + true false @@ -644,7 +653,9 @@ package - target/dataverse-${project.version}/WEB-INF/lib/aws-java-sdk-bundle-${aws.version}.jar/META-INF + + target/dataverse-${project.version}/WEB-INF/lib/aws-java-sdk-bundle-${aws.version}.jar/META-INF + javamail.providers @@ -659,7 +670,9 @@ package - target/dataverse-${project.version}.war/WEB-INF/lib/aws-java-sdk-bundle-${aws.version}.jar/META-INF + + target/dataverse-${project.version}.war/WEB-INF/lib/aws-java-sdk-bundle-${aws.version}.jar/META-INF + javamail.providers @@ -671,7 +684,7 @@ maven-surefire-plugin - 2.22.0 + 3.0.0-M1 ${testsToExclude} @@ -690,20 +703,20 @@ - - dev - - - - true - - - edu.harvard.iq.dataverse.NonEssentialTests - - - - all-unit-tests - - + + dev + + + + true + + + edu.harvard.iq.dataverse.NonEssentialTests + + + + all-unit-tests + + diff --git a/src/main/java/edu/harvard/iq/dataverse/flyway/StartupFlywayMigrator.java b/src/main/java/edu/harvard/iq/dataverse/flyway/StartupFlywayMigrator.java new file mode 100644 index 00000000000..0cddb266f44 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/flyway/StartupFlywayMigrator.java @@ -0,0 +1,28 @@ +package edu.harvard.iq.dataverse.flyway; + +import org.flywaydb.core.Flyway; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import javax.ejb.Singleton; +import javax.ejb.Startup; +import javax.sql.DataSource; + +@Startup +@Singleton +class StartupFlywayMigrator { + + @Resource(lookup = "jdbc/VDCNetDS") + private DataSource dataSource; + + @PostConstruct + void migrateDatabase() { + + Flyway flyway = Flyway.configure() + .dataSource(dataSource) + .baselineOnMigrate(true) + .load(); + + flyway.migrate(); + } +} diff --git a/src/main/resources/db/migration/V1__flyway_schema_baseline.sql b/src/main/resources/db/migration/V1__flyway_schema_baseline.sql new file mode 100644 index 00000000000..e69de29bb2d From 71d3be7e2722bdafb8ce9c0b8e8cfb22a2ba0e68 Mon Sep 17 00:00:00 2001 From: Karol Kulik Date: Tue, 27 Nov 2018 14:41:44 +0100 Subject: [PATCH 2/9] Updated text tutorial about upgrading schema. --- .../source/developers/sql-upgrade-scripts.rst | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index d50ecc0d72e..92c66803dbc 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,40 +2,38 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL upgrade scripts when needed and communicate with your fellow developers about applying those scripts. +The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL when needed and communicate with your fellow developers about making those SQLs. .. contents:: |toctitle| :local: -Location of SQL Upgrade Scripts -------------------------------- +Location of SQL Upgrade Scripts and Flyway SQLs +----------------------------------------------- -``scripts/database/upgrades`` is the directory where we keep or SQL upgrade scripts. +``scripts/database/upgrades`` is the directory where we keep SQL upgrade scripts - legacy way of upgrading schema. +``resources/db/migration`` is the directory where we keep migration SQLs. -How to Determine if You Need to Create or Update a SQL Upgrade Script ---------------------------------------------------------------------- +SQL naming conventions +---------------------- +In order to make the migrations work you must follow the naming conventions of the flyway - https://flywaydb.org/documentation/migrations#sql-based-migrations -If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create or update a SQL upgrade script. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. +How to Determine if You Need to Create SQL +------------------------------------------ -If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create or update a SQL upgrade script. +If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. -How to Create or Update a SQL Upgrade Script --------------------------------------------- +If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create SQL. -We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. - -First, check https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades to see if a SQL upgrade script for the next release already exists. For example, if the current release is 4.9.4 and the next release will be 4.10, the script will be named ``upgrade_v4.9.4_to_v4.10.sql``. If the script exists, just add your changes to the bottom of it. +How to Create SQL +----------------- -If no SQL upgrade script exists, look at https://github.com/IQSS/dataverse/milestones to figure out the name of the next milestone and create a script using the naming convention above. - -As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. -Please note that we are aware of the problem of merge conflicts in the SQL update script as well as how the next version number can change at any time. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL upgrade scripts. +First, check https://github.com/IQSS/dataverse/tree/develop/src/main/resources/db/migration to see if a SQL numbers are not colliding with your's. -Communicating the Need to Run SQL Updates ------------------------------------------ +As with any task related to Dataverse development, if you need any help writing SQL, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. -If you have made a pull request that contains SQL updates and that pull request is merged into the "develop" branch, you are responsible for communicating to other developers that when then pull the latest code from "develop" they must run your SQL updates. Post a message to the "dataverse-dev" mailing list at https://groups.google.com/forum/#!forum/dataverse-dev +Please note that we are aware of the problem of merge conflicts in the SQLs as well as how the next number can change at any time. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL upgrade scripts. ---- From eca407bd635201b2657ff38304226e1c863cce50 Mon Sep 17 00:00:00 2001 From: Karol Kulik Date: Tue, 27 Nov 2018 14:43:58 +0100 Subject: [PATCH 3/9] Updated text tutorial about upgrading schema. --- .../source/developers/sql-upgrade-scripts.rst | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index d50ecc0d72e..92c66803dbc 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,40 +2,38 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL upgrade scripts when needed and communicate with your fellow developers about applying those scripts. +The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL when needed and communicate with your fellow developers about making those SQLs. .. contents:: |toctitle| :local: -Location of SQL Upgrade Scripts -------------------------------- +Location of SQL Upgrade Scripts and Flyway SQLs +----------------------------------------------- -``scripts/database/upgrades`` is the directory where we keep or SQL upgrade scripts. +``scripts/database/upgrades`` is the directory where we keep SQL upgrade scripts - legacy way of upgrading schema. +``resources/db/migration`` is the directory where we keep migration SQLs. -How to Determine if You Need to Create or Update a SQL Upgrade Script ---------------------------------------------------------------------- +SQL naming conventions +---------------------- +In order to make the migrations work you must follow the naming conventions of the flyway - https://flywaydb.org/documentation/migrations#sql-based-migrations -If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create or update a SQL upgrade script. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. +How to Determine if You Need to Create SQL +------------------------------------------ -If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create or update a SQL upgrade script. +If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. -How to Create or Update a SQL Upgrade Script --------------------------------------------- +If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create SQL. -We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. - -First, check https://github.com/IQSS/dataverse/tree/develop/scripts/database/upgrades to see if a SQL upgrade script for the next release already exists. For example, if the current release is 4.9.4 and the next release will be 4.10, the script will be named ``upgrade_v4.9.4_to_v4.10.sql``. If the script exists, just add your changes to the bottom of it. +How to Create SQL +----------------- -If no SQL upgrade script exists, look at https://github.com/IQSS/dataverse/milestones to figure out the name of the next milestone and create a script using the naming convention above. - -As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. -Please note that we are aware of the problem of merge conflicts in the SQL update script as well as how the next version number can change at any time. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL upgrade scripts. +First, check https://github.com/IQSS/dataverse/tree/develop/src/main/resources/db/migration to see if a SQL numbers are not colliding with your's. -Communicating the Need to Run SQL Updates ------------------------------------------ +As with any task related to Dataverse development, if you need any help writing SQL, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. -If you have made a pull request that contains SQL updates and that pull request is merged into the "develop" branch, you are responsible for communicating to other developers that when then pull the latest code from "develop" they must run your SQL updates. Post a message to the "dataverse-dev" mailing list at https://groups.google.com/forum/#!forum/dataverse-dev +Please note that we are aware of the problem of merge conflicts in the SQLs as well as how the next number can change at any time. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL upgrade scripts. ---- From 5f42f44b01520b5df0af570c9c2fef27a9c6822c Mon Sep 17 00:00:00 2001 From: Karol Kulik Date: Wed, 28 Nov 2018 13:33:27 +0100 Subject: [PATCH 4/9] Updated text tutorial about upgrading schema. --- doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 92c66803dbc..11b9edd76f9 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -33,7 +33,7 @@ First, check https://github.com/IQSS/dataverse/tree/develop/src/main/resources/d As with any task related to Dataverse development, if you need any help writing SQL, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. -Please note that we are aware of the problem of merge conflicts in the SQLs as well as how the next number can change at any time. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL upgrade scripts. +Please note that we are aware of the problem of merge conflicts in the SQL numbers. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQLs. ---- From 85fefecb89ab82959463f6f055159d133c63f557 Mon Sep 17 00:00:00 2001 From: Karol Date: Thu, 14 Feb 2019 10:27:02 +0100 Subject: [PATCH 5/9] changed documentation, added validation. --- .../source/developers/sql-upgrade-scripts.rst | 8 ++++---- pom.xml | 5 +++-- .../iq/dataverse/flyway/StartupFlywayMigrator.java | 11 +++++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 11b9edd76f9..47ff072aa53 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,16 +2,16 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL when needed and communicate with your fellow developers about making those SQLs. +The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL when needed and communicate with your fellow developers about making those SQL scripts. .. contents:: |toctitle| :local: -Location of SQL Upgrade Scripts and Flyway SQLs +Location of SQL Upgrade Scripts and Flyway SQL scripts ----------------------------------------------- ``scripts/database/upgrades`` is the directory where we keep SQL upgrade scripts - legacy way of upgrading schema. -``resources/db/migration`` is the directory where we keep migration SQLs. +``resources/db/migration`` is the directory where we keep migration SQL scripts. SQL naming conventions ---------------------- @@ -33,7 +33,7 @@ First, check https://github.com/IQSS/dataverse/tree/develop/src/main/resources/d As with any task related to Dataverse development, if you need any help writing SQL, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. -Please note that we are aware of the problem of merge conflicts in the SQL numbers. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQLs. +Please note that we are aware of the problem of merge conflicts in the SQL numbers. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL scripts. ---- diff --git a/pom.xml b/pom.xml index a2a979a115d..3740e51540c 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,7 @@ 5.3.1 1.3.1 2.22.0 + 5.2.4 maven-surefire-plugin - 3.0.0-M1 + 2.22.0 ${testsToExclude} From d458c8349dcf0cf76b9da04268e41190f5e58aa4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 7 Mar 2019 16:15:40 -0500 Subject: [PATCH 7/9] move 4.11 to 4.12 SQL script to Flyway directory, docs #5344 --- doc/release-notes/README.md | 4 +-- .../source/developers/making-releases.rst | 3 +- .../source/developers/sql-upgrade-scripts.rst | 33 +++++++++---------- .../V1.1__5513-database-variablemetadata.sql | 0 4 files changed, 19 insertions(+), 21 deletions(-) rename scripts/database/upgrades/upgrade_v4.11_to_v4.12.sql => src/main/resources/db/migration/V1.1__5513-database-variablemetadata.sql (100%) diff --git a/doc/release-notes/README.md b/doc/release-notes/README.md index 8c2d5748034..17334b3f11e 100644 --- a/doc/release-notes/README.md +++ b/doc/release-notes/README.md @@ -3,10 +3,10 @@ doc/sphinx-guides/source/developers/making-releases.rst documents the official process for making release notes but as indicated there, we are experimenting with a process with the following goals: - As a developer, I want to express in my pull request when an addition to the release notes will be necessary. -- As a developer, I want to be aware of changes that should be made to my dev environment after a pull request has been merged. I already know to look in `scripts/database/upgrades` if I pull the latest code from the "develop" branch for updates as described in doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst but I want a place to look for non-SQL updates that are required. These could be Solr schema changes or curl commands to reload metadata blocks, for example. +- As a developer, I want to be aware of changes that should be made to my dev environment after a pull request has been merged. These could be Solr schema changes or curl commands to reload metadata blocks, for example. # release-notes directory process - Create a Markdown file named after your branch (assuming your branch starts with an issue number as requested in doc/sphinx-guides/source/developers/version-control.rst) such as "5053-apis-custom-homepage.md". -- In the file you created, give instructions for non-SQL upgrade steps that must be taken to run the branch in your pull request. Examples include Solr schema updates or reloading metadata blocks. +- In the file you created, give instructions for non-SQL upgrade steps that must be taken to run the branch in your pull request. Examples include Solr schema updates or reloading metadata blocks. (SQL updates are handled separately as indicated in doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst.) - At release time, gather all the files into final release notes and make a `git rm` commit to delete them to prevent clutter. diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 771b682385f..5db32efb9c9 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -62,7 +62,7 @@ Create a draft release at https://github.com/IQSS/dataverse/releases/new - The "tag version" and "title" should be the number of the milestone with a "v" in front (i.e. v4.6.2). - For the description, follow previous examples at https://github.com/IQSS/dataverse/releases -Please note that the current process involves copying and pasting a running Google doc into release notes but we are conducting an experiment whereby developers can express the need for an addition to release notes by creating a file in ``/doc/release-notes`` containing the name of the issue they're working on. Perhaps the name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``. To avoid accumulating many stale files over time, when a release is cut these files should probably be removed with ``git rm``. This experiment may help inform a future experiment having to do with improvements to our process for writing SQL upgrade scripts. See the :doc:`sql-upgrade-scripts` section for more on this topic. +Please note that the current process involves copying and pasting a running Google doc into release notes but we are conducting an experiment whereby developers can express the need for an addition to release notes by creating a file in ``/doc/release-notes`` containing the name of the issue they're working on. Perhaps the name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``. To avoid accumulating many stale files over time, when a release is cut these files should probably be removed with ``git rm``. Make Artifacts Available for Download ------------------------------------- @@ -71,7 +71,6 @@ Upload the following artifacts to the draft release you created: - war file (``mvn package`` from Jenkins) - installer (``cd scripts/installer && make``) -- database migration script (see also the :doc:`sql-upgrade-scripts` section) - other files as needed, such as an updated Solr schema Publish Release diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 47ff072aa53..1888fb270fd 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -2,38 +2,37 @@ SQL Upgrade Scripts =================== -The database schema for Dataverse is constantly evolving. As other developers make changes to the database schema you will need to keep up with these changes to have your development environment in working order. Additionally, as you make changes to the database schema, you must write SQL when needed and communicate with your fellow developers about making those SQL scripts. +The database schema for Dataverse is constantly evolving and we have adopted a tool called Flyway to help keep your development environment up to date and in working order. As you make changes to the database schema (changes to ``@Entity`` classes), you must write SQL upgrade scripts when needed and follow Flyway file naming conventions. .. contents:: |toctitle| :local: -Location of SQL Upgrade Scripts and Flyway SQL scripts ------------------------------------------------ +Location of SQL Upgrade Scripts +------------------------------- -``scripts/database/upgrades`` is the directory where we keep SQL upgrade scripts - legacy way of upgrading schema. -``resources/db/migration`` is the directory where we keep migration SQL scripts. +``src/main/resources/db/migration`` is the directory where we keep SQL upgrade scripts for Flyway to find. -SQL naming conventions ----------------------- -In order to make the migrations work you must follow the naming conventions of the flyway - https://flywaydb.org/documentation/migrations#sql-based-migrations +In the past (before adopting Flyway) we used to keep SQL upgrade scripts in ``scripts/database/upgrades``. These scripts can still be used as reference but no new scripts should be added there. -How to Determine if You Need to Create SQL ------------------------------------------- +How to Determine if You Need to Create a SQL Upgrade Script +----------------------------------------------------------- -If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. +If you are creating a new database table (which maps to an ``@Entity`` in JPA), you do not need to create or update a SQL upgrade script. The reason for this is that we use ``create-tables`` in ``src/main/resources/META-INF/persistence.xml`` so that new tables are automatically created by Glassfish when you deploy your war file. -If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create SQL. +If you are doing anything other than creating a new database table such as adding a column to an existing table, you must create or update a SQL upgrade script. -How to Create SQL ------------------ +How to Create a SQL Upgrade Script +---------------------------------- We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. -First, check https://github.com/IQSS/dataverse/tree/develop/src/main/resources/db/migration to see if a SQL numbers are not colliding with your's. +First, compare https://github.com/IQSS/dataverse/tree/develop/src/main/resources/db/migration to your local ``src/main/resources/db/migration`` directory to make sure you have all of the SQL migration scripts. If you are missing any, merge the "develop" branch into your branch. -As with any task related to Dataverse development, if you need any help writing SQL, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +Create a new file called something like ``V1.1__5513-database-variablemetadata.sql`` that has a unique file name and follows the Flyway file naming conventions at https://flywaydb.org/documentation/migrations#naming . For the "description" you should include the GitHub issue you are working on, as in the example above. -Please note that we are aware of the problem of merge conflicts in the SQL numbers. Please see the :doc:`making-releases` section for how we are running an experiment having to do with release notes that might help inform an improvement of our process for developing SQL scripts. +The SQL migration script you wrote will be part of the war file and executed when the war file is deployed. To see a history of Flyway database migrations that have been applied, look at the ``flyway_schema_history`` table. + +As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. ---- diff --git a/scripts/database/upgrades/upgrade_v4.11_to_v4.12.sql b/src/main/resources/db/migration/V1.1__5513-database-variablemetadata.sql similarity index 100% rename from scripts/database/upgrades/upgrade_v4.11_to_v4.12.sql rename to src/main/resources/db/migration/V1.1__5513-database-variablemetadata.sql From 0c108973f47a6c8387641dd5f91c13e1f5b52be7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 14 Mar 2019 09:42:57 -0400 Subject: [PATCH 8/9] use previous Dataverse version in SQL script name #5344 --- .../source/developers/sql-upgrade-scripts.rst | 14 +++++++++++--- ...l => V4.11__5513-database-variablemetadata.sql} | 0 2 files changed, 11 insertions(+), 3 deletions(-) rename src/main/resources/db/migration/{V1.1__5513-database-variablemetadata.sql => V4.11__5513-database-variablemetadata.sql} (100%) diff --git a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst index 1888fb270fd..07205a5d365 100644 --- a/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst +++ b/doc/sphinx-guides/source/developers/sql-upgrade-scripts.rst @@ -26,14 +26,22 @@ How to Create a SQL Upgrade Script We assume you have already read the :doc:`version-control` section and have been keeping your feature branch up to date with the "develop" branch. -First, compare https://github.com/IQSS/dataverse/tree/develop/src/main/resources/db/migration to your local ``src/main/resources/db/migration`` directory to make sure you have all of the SQL migration scripts. If you are missing any, merge the "develop" branch into your branch. - -Create a new file called something like ``V1.1__5513-database-variablemetadata.sql`` that has a unique file name and follows the Flyway file naming conventions at https://flywaydb.org/documentation/migrations#naming . For the "description" you should include the GitHub issue you are working on, as in the example above. +Create a new file called something like ``V4.11__5513-database-variablemetadata.sql`` in the ``src/main/resources/db/migration`` directory. Use the previously released version (4.11 in the example above) rather than what we guess will be the next released version (which is often uncertain). For the "description" you should the name of your branch, which should include the GitHub issue you are working on, as in the example above. To read more about Flyway file naming conventions, see https://flywaydb.org/documentation/migrations#naming The SQL migration script you wrote will be part of the war file and executed when the war file is deployed. To see a history of Flyway database migrations that have been applied, look at the ``flyway_schema_history`` table. As with any task related to Dataverse development, if you need any help writing SQL upgrade scripts, please reach out using any of the channels mentioned under "Getting Help" in the :doc:`intro` section. +Troubleshooting +--------------- + +Renaming SQL Upgrade Scripts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Please note that if you need to rename your script (because a new version of Dataverse was released, for example), you will see the error "FlywayException: Validate failed: Detected applied migration not resolved locally" when you attempt to deploy and deployment will fail. + +To resolve this problem, delete the old migration from the ``flyway_schema_history`` table and attempt to redeploy. + ---- Previous: :doc:`version-control` | Next: :doc:`testing` diff --git a/src/main/resources/db/migration/V1.1__5513-database-variablemetadata.sql b/src/main/resources/db/migration/V4.11__5513-database-variablemetadata.sql similarity index 100% rename from src/main/resources/db/migration/V1.1__5513-database-variablemetadata.sql rename to src/main/resources/db/migration/V4.11__5513-database-variablemetadata.sql From 3b07490bb308e47e99d4ca6c529a1ac59bc00619 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 14 Mar 2019 09:56:30 -0400 Subject: [PATCH 9/9] note new flyway table on upgrading page #5344 --- doc/sphinx-guides/source/installation/upgrading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/upgrading.rst b/doc/sphinx-guides/source/installation/upgrading.rst index 66c491a3e87..a180d6492a5 100644 --- a/doc/sphinx-guides/source/installation/upgrading.rst +++ b/doc/sphinx-guides/source/installation/upgrading.rst @@ -7,7 +7,7 @@ Upgrading When upgrading within Dataverse 4.x, you will need to follow the upgrade instructions for each intermediate version. -Upgrades always involve deploying the latest war file but may also include running SQL scripts and updating the schema used by Solr. +Upgrades always involve deploying the latest war file but may also include updating the schema used by Solr or other manual steps. Running database migration scripts was once required but this has been automated (see the ``flyway_schema_history`` database table to see migrations that have been run). Please consult the release notes associated with each release at https://github.com/IQSS/dataverse/releases for more information.