diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml
new file mode 100644
index 00000000000..7a56b8c2f7d
--- /dev/null
+++ b/.github/workflows/shellspec.yml
@@ -0,0 +1,83 @@
+name: "Shellspec"
+on:
+ push:
+ paths:
+ - tests/shell/**
+ - conf/solr/**
+ # add more when more specs are written relying on data
+ pull_request:
+ paths:
+ - tests/shell/**
+ - conf/solr/**
+ # add more when more specs are written relying on data
+env:
+ SHELLSPEC_VERSION: 0.28.1
+jobs:
+ shellcheck:
+ name: Shellcheck
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: shellcheck
+ uses: reviewdog/action-shellcheck@v1
+ with:
+ github_token: ${{ secrets.github_token }}
+ reporter: github-pr-review # Change reporter.
+ fail_on_error: true
+ exclude: "./tests/shell/*"
+ shellspec-ubuntu:
+ name: "Ubuntu"
+ runs-on: ubuntu-latest
+ steps:
+ - name: Install shellspec
+ run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes
+ - uses: actions/checkout@v2
+ - name: Run Shellspec
+ run: |
+ cd tests/shell
+ shellspec
+ shellspec-centos7:
+ name: "CentOS 7"
+ runs-on: ubuntu-latest
+ container:
+ image: centos:7
+ steps:
+ - uses: actions/checkout@v2
+ - name: Install shellspec
+ run: |
+ curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share
+ ln -s /usr/share/shellspec/shellspec /usr/bin/shellspec
+ - name: Install dependencies
+ run: yum install -y ed
+ - name: Run shellspec
+ run: |
+ cd tests/shell
+ shellspec
+ shellspec-rocky8:
+ name: "RockyLinux 8"
+ runs-on: ubuntu-latest
+ container:
+ image: rockylinux/rockylinux:8
+ steps:
+ - uses: actions/checkout@v2
+ - name: Install shellspec
+ run: |
+ curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share
+ ln -s /usr/share/shellspec/shellspec /usr/bin/shellspec
+ - name: Install dependencies
+ run: dnf install -y ed bc diffutils
+ - name: Run shellspec
+ run: |
+ cd tests/shell
+ shellspec
+ shellspec-macos:
+ name: "MacOS"
+ runs-on: macos-10.15
+ steps:
+ - name: Install shellspec
+ run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes
+ - uses: actions/checkout@v2
+ - name: Run Shellspec
+ run: |
+ cd tests/shell
+ /Users/runner/.local/bin/shellspec
diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh
index 98cbc2eec93..2c332414e94 100755
--- a/conf/docker-aio/1prep.sh
+++ b/conf/docker-aio/1prep.sh
@@ -6,7 +6,6 @@
mkdir -p testdata/doc/sphinx-guides/source/_static/util/
cp ../solr/8.8.1/schema*.xml testdata/
cp ../solr/8.8.1/solrconfig.xml testdata/
-cp ../solr/8.8.1/updateSchemaMDB.sh testdata/
cp ../jhove/jhove.conf testdata/
cp ../jhove/jhoveConfig.xsd testdata/
cd ../../
diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile
index 515f69b8c55..e08e1d56749 100644
--- a/conf/docker-aio/c8.dockerfile
+++ b/conf/docker-aio/c8.dockerfile
@@ -64,7 +64,6 @@ COPY dv/install/ /opt/dv/
COPY install.bash /opt/dv/
COPY entrypoint.bash /opt/dv/
COPY testdata /opt/dv/testdata
-COPY testdata/updateSchemaMDB.sh /opt/dv/testdata/
COPY testscripts/* /opt/dv/testdata/
COPY setupIT.bash /opt/dv
WORKDIR /opt/dv
diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml
index 3d46a8336be..d8c19d45a81 100644
--- a/conf/solr/8.8.1/schema.xml
+++ b/conf/solr/8.8.1/schema.xml
@@ -228,8 +228,181 @@
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -290,8 +463,22 @@
-
-
+
+
+
@@ -451,7 +638,7 @@
-
+
diff --git a/conf/solr/8.8.1/schema_dv_mdb_fields.xml b/conf/solr/8.8.1/schema_dv_mdb_fields.xml
deleted file mode 100644
index 3f844c6183c..00000000000
--- a/conf/solr/8.8.1/schema_dv_mdb_fields.xml
+++ /dev/null
@@ -1,161 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/conf/solr/8.8.1/update-fields.sh b/conf/solr/8.8.1/update-fields.sh
new file mode 100755
index 00000000000..49ea8151c77
--- /dev/null
+++ b/conf/solr/8.8.1/update-fields.sh
@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
+# This script will
+# 1. take a file (or read it from STDIN) with all and definitions
+# 2. and replace the sections between the include guards with those in a given
+# schema.xml file
+# The script validates the presence, uniqueness and order of the include guards.
+#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### ####
+
+
+### Variables
+# Internal use only (fork to change)
+VERSION="0.1"
+INPUT=""
+FIELDS=""
+COPY_FIELDS=""
+TRIGGER_CHAIN=0
+ED_DELETE_FIELDS="'a+,'b-d"
+ED_DELETE_COPYFIELDS="'a+,'b-d"
+
+SOLR_SCHEMA_FIELD_BEGIN_MARK="SCHEMA-FIELDS::BEGIN"
+SOLR_SCHEMA_FIELD_END_MARK="SCHEMA-FIELDS::END"
+SOLR_SCHEMA_COPYFIELD_BEGIN_MARK="SCHEMA-COPY-FIELDS::BEGIN"
+SOLR_SCHEMA_COPYFIELD_END_MARK="SCHEMA-COPY-FIELDS::END"
+MARKS_ORDERED="${SOLR_SCHEMA_FIELD_BEGIN_MARK} ${SOLR_SCHEMA_FIELD_END_MARK} ${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK} ${SOLR_SCHEMA_COPYFIELD_END_MARK}"
+
+### Common functions
+function error {
+ echo "ERROR:" "$@" >&2
+ exit 2
+}
+
+function exists {
+ type "$1" >/dev/null 2>&1 && return 0
+ ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0; done; return 1 )
+}
+
+function usage {
+ cat << EOF
+$(basename "$0") ${VERSION}
+Usage: $(basename "$0") [-hp] [ schema file ] [ source file ]
+
+-h Print usage (this text)
+-p Chained printing: write all metadata schema related
+ and present in Solr XML to stdout
+
+Provide target Solr Schema XML via argument or \$SCHEMA env var.
+
+Provide source file via argument, \$SOURCE env var or piped input
+(wget/curl, chained). Source file = "-" means read STDIN.
+EOF
+ exit 0
+}
+
+### Options
+while getopts ":hp" opt; do
+ case $opt in
+ h) usage ;;
+ p) TRIGGER_CHAIN=1 ;;
+ \?) echo "Invalid option: -$OPTARG" >&2; exit 1 ;;
+ :) echo "Option -$OPTARG requires an argument." >&2; exit 1 ;;
+ esac
+done
+
+# Check for ed and bc being present
+exists ed || error "Please ensure ed, bc, sed + awk are installed"
+exists bc || error "Please ensure ed, bc, sed + awk are installed"
+exists awk || error "Please ensure ed, bc, sed + awk are installed"
+exists sed || error "Please ensure ed, bc, sed + awk are installed"
+
+# remove all the parsed options
+shift $((OPTIND-1))
+
+# User overrideable locations
+SCHEMA=${SCHEMA:-${1:-schema.xml}}
+SOURCE=${SOURCE:-${2:-"-"}}
+
+
+### VERIFY SCHEMA FILE EXISTS AND CONTAINS INCLUDE GUARDS ###
+# Check for schema file & writeable
+if [ ! -w "${SCHEMA}" ]; then
+ error "Cannot find or write to a XML schema at ${SCHEMA}"
+else
+ # Check schema file for include guards
+ CHECKS=$(
+ for MARK in ${MARKS_ORDERED}
+ do
+ grep -c "${MARK}" "${SCHEMA}" || error "Missing ${MARK} from ${SCHEMA}"
+ done
+ )
+
+ # Check guards are unique (count occurrences and sum calc via bc)
+ # Note: fancy workaround to re-add closing \n on Linux & MacOS or no calculation
+ [ "$( (echo -n "${CHECKS}" | tr '\n' '+' ; echo ) | bc)" -eq 4 ] || \
+ error "Some include guards are not unique in ${SCHEMA}"
+
+ # Check guards are in order (line number comparison via bc tricks)
+ CHECKS=$(
+ for MARK in ${MARKS_ORDERED}
+ do
+ grep -n "${MARK}" "${SCHEMA}" | cut -f 1 -d ":"
+ done
+ )
+ # Actual comparison of line numbers
+ echo "${CHECKS}" | tr '\n' '<' | awk -F'<' '{ if ($1 < $2 && $2 < $3 && $3 < $4) {exit 0} else {exit 1} }' || \
+ error "Include guards are not in correct order in ${SCHEMA}"
+
+ # Check guards are exclusively in their lines
+ # (no field or copyField on same line)
+ for MARK in ${MARKS_ORDERED}
+ do
+ grep "${MARK}" "${SCHEMA}" | grep -q -v -e '\(" "${SOURCE}" | sed -e 's#^\s\+##' -e 's#\s\+$##' || true)
+
+
+### DATA HANDLING ###
+# Split input into different types
+if [ -z "${INPUT}" ]; then
+ error "No or in input"
+else
+ # Check for definitions (if nomatch, avoid failing pipe)
+ FIELDS=$(mktemp)
+ echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${FIELDS}" || true
+ # If file actually contains output, write to schema
+ if [ -s "${FIELDS}" ]; then
+ # Use an ed script to replace all
+ cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}"
+H
+# Mark field begin as 'a'
+/${SOLR_SCHEMA_FIELD_BEGIN_MARK}/ka
+# Mark field end as 'b'
+/${SOLR_SCHEMA_FIELD_END_MARK}/kb
+# Delete all between lines a and b
+${ED_DELETE_FIELDS}
+# Read fields file and paste after line a
+'ar ${FIELDS}
+# Write fields to schema
+w
+q
+EOF
+ fi
+ rm "${FIELDS}"
+
+ # Check for definitions (if nomatch, avoid failing pipe)
+ COPY_FIELDS=$(mktemp)
+ echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${COPY_FIELDS}" || true
+ # If file actually contains output, write to schema
+ if [ -s "${COPY_FIELDS}" ]; then
+ # Use an ed script to replace all , filter comments (BSD ed does not support comments)
+ cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}"
+H
+# Mark copyField begin as 'a'
+/${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}/ka
+# Mark copyField end as 'b'
+/${SOLR_SCHEMA_COPYFIELD_END_MARK}/kb
+# Delete all between lines a and b
+${ED_DELETE_COPYFIELDS}
+# Read fields file and paste after line a
+'ar ${COPY_FIELDS}
+# Write copyFields to schema
+w
+q
+EOF
+ fi
+ rm "${COPY_FIELDS}"
+fi
+
+
+### CHAINING OUTPUT
+# Scripts following this one might want to use the field definitions now present
+if [ "${TRIGGER_CHAIN}" -eq 1 ]; then
+ grep -A1000 "${SOLR_SCHEMA_FIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_FIELD_END_MARK}"
+ grep -A1000 "${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_COPYFIELD_END_MARK}"
+fi
diff --git a/conf/solr/8.8.1/updateSchemaMDB.sh b/conf/solr/8.8.1/updateSchemaMDB.sh
deleted file mode 100755
index 0044f15c7cd..00000000000
--- a/conf/solr/8.8.1/updateSchemaMDB.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# This script updates the and schema configuration necessary to properly
-# index custom metadata fields in Solr.
-# 1. Retrieve from Dataverse API endpoint
-# 2. Parse and write Solr schema files (which might replace the included files)
-# 3. Reload Solr
-#
-# List of variables:
-# ${DATAVERSE_URL}: URL to Dataverse. Defaults to http://localhost:8080
-# ${SOLR_URL}: URL to Solr. Defaults to http://localhost:8983
-# ${UNBLOCK_KEY}: File path to secret or unblock key as string. Only necessary on k8s or when you secured your installation.
-# ${TARGET}: Directory where to write the XML files. Defaults to /tmp
-#
-# Programs used (need to be available on your PATH):
-# coreutils: mktemp, csplit
-# curl
-
-usage() {
- echo "usage: updateSchemaMDB.sh [options]"
- echo "options:"
- echo " -d Dataverse URL, defaults to http://localhost:8080"
- echo " -h Show this help text"
- echo " -s Solr URL, defaults to http://localhost:8983"
- echo " -t Directory where to write the XML files. Defaults to /tmp"
- echo " -u Dataverse unblock key either as key string or path to keyfile"
-}
-
-### Init (with sane defaults)
-DATAVERSE_URL=${DATAVERSE_URL:-"http://localhost:8080"}
-SOLR_URL=${SOLR_URL:-"http://localhost:8983"}
-TARGET=${TARGET:-"/tmp"}
-UNBLOCK_KEY=${UNBLOCK_KEY:-""}
-
-# if cmdline args are given, override any env var setting (or defaults)
-while getopts ":d:hs:t:u:" opt
-do
- case $opt in
- d) DATAVERSE_URL=${OPTARG};;
- h) usage; exit 0;;
- s) SOLR_URL=${OPTARG};;
- t) TARGET=${OPTARG};;
- u) UNBLOCK_KEY=${OPTARG};;
- :) echo "Missing option argument for -${OPTARG}. Use -h for help." >&2; exit 1;;
- \?) echo "Unknown option -${OPTARG}." >&2; usage; exit 1;;
- esac
-done
-
-# Special handling of unblock key depending on referencing a secret file or key in var
-if [ ! -z "${UNBLOCK_KEY}" ]; then
- if [ -f "${UNBLOCK_KEY}" ]; then
- UNBLOCK_KEY="?unblock-key=$(cat ${UNBLOCK_KEY})"
- else
- UNBLOCK_KEY="?unblock-key=${UNBLOCK_KEY}"
- fi
-fi
-
-### Retrieval
-echo "Retrieve schema data from ${DATAVERSE_URL}/api/admin/index/solr/schema"
-TMPFILE=`mktemp`
-curl -f -sS "${DATAVERSE_URL}/api/admin/index/solr/schema${UNBLOCK_KEY}" > $TMPFILE
-
-### Fail gracefull if Dataverse is not ready yet.
-if [[ "`wc -l ${TMPFILE}`" < "3" ]]; then
- echo "Dataverse responded with empty file. When running on K8s: did you bootstrap yet?"
- exit 123
-fi
-
-### Processing
-echo "Writing ${TARGET}/schema_dv_mdb_fields.xml"
-echo "" > ${TARGET}/schema_dv_mdb_fields.xml
-cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_fields.xml
-echo "" >> ${TARGET}/schema_dv_mdb_fields.xml
-
-echo "Writing ${TARGET}/schema_dv_mdb_copies.xml"
-echo "" > ${TARGET}/schema_dv_mdb_copies.xml
-cat ${TMPFILE} | grep ".*> ${TARGET}/schema_dv_mdb_copies.xml
-echo "" >> ${TARGET}/schema_dv_mdb_copies.xml
-
-rm ${TMPFILE}*
-
-### Reloading
-echo "Triggering Solr RELOAD at ${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1"
-curl -f -sS "${SOLR_URL}/solr/admin/cores?action=RELOAD&core=collection1"
diff --git a/doc/release-notes/7903-solr-config.md b/doc/release-notes/7903-solr-config.md
new file mode 100644
index 00000000000..1c4ed47d40a
--- /dev/null
+++ b/doc/release-notes/7903-solr-config.md
@@ -0,0 +1,30 @@
+## Mitigate Solr Schema Management Problems
+
+With [release 5.5](https://github.com/IQSS/dataverse/releases/tag/v5.5), the `` definitions had been
+reincluded into `schema.xml` to fix searching for datasets.
+
+This release includes a final update to `schema.xml` and an updated script `update-fields.sh` to manage your
+custom metadata fields in the future. (It might get used for other purposes in the future, too.) The broken script
+`updateSchemaMDB.sh` has been removed.
+
+Please replace your schema.xml with the one provided to make sure the new script can do its magic.
+If you do not use any custom metadata blocks, that's it. Else, read on.
+
+To include your custom metadata fields after updating schema.xml, you can use a simple `curl` command. (Please download
+the script before or use it from the extracted installer.)
+
+```
+curl "https:///api/admin/index/solr/schema" | update-fields.sh conf/schema.xml
+```
+
+Please adapt the above to point to your Dataverse installation and to the correct `schema.xml` file in your Solr
+installation. (See the [installation guide](https://guides.dataverse.org/en/latest/installation/prerequisites.html#installing-solr)
+for some hints about usual places or use `find / -name schema.xml`)
+
+After upgrade, you need to restart (downtime!) or reload Solr (OK while running):
+```
+curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"
+```
+(Please adapt to your installations details. Should work as is for most.)
+
+TODO: if we change any schemas in this release, a hint about reindexing might be necessary. Else delete this TODO.
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 9e62b66febb..c307adb56af 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -639,32 +639,38 @@ Updating the Solr Schema
Once you have enabled a new metadata block you should be able to see the new fields in the GUI but before you can save
the dataset, you must add additional fields to your Solr schema.
-An API endpoint of your Dataverse installation provides you with a generated set of all fields that need to be added to the Solr schema
-configuration, including any enabled metadata schemas:
+An API endpoint of your Dataverse installation provides you with a generated set of all fields that need to be added to
+the Solr schema configuration, including any enabled metadata schemas:
-``curl http://localhost:8080/api/admin/index/solr/schema``
+``curl "http://localhost:8080/api/admin/index/solr/schema"``
+
+You can use :download:`update-fields.sh <../../../../conf/solr/8.8.1/update-fields.sh>` to easily add these to the
+Solr schema you installed for your Dataverse installation.
+
+The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of
+the Installation Guide for a suggested location on disk for the Solr schema file.)
+
+You can either pipe the downloaded schema to the script or provide the file as an argument. (We recommended you to take
+a look at usage output of ``update-fields.sh -h``)
+
+.. code-block::
+ :caption: Example usage of ``update-fields.sh``
+
+ curl "http://localhost:8080/api/admin/index/solr/schema" | update-fields.sh /usr/local/solr/server/solr/collection1/conf/schema.xml
-For convenience and automation you can download and consider running :download:`updateSchemaMDB.sh <../../../../conf/solr/8.8.1/updateSchemaMDB.sh>`. It uses the API endpoint above and writes schema files to the filesystem (so be sure to run it on the Solr server itself as the Unix user who owns the Solr files) and then triggers a Solr reload.
-Due to `an issue with schema.xml including the generated schema_dv_mdb_copies.xml file `_ additional steps are currently needed. Once schema_dv_mdb_copies.xml has been generated by the script, you'll need to copy/paste the set of elements in the generated file into schema.xml manually and then restart solr (or trigger a Solr reload).
+You will need to reload your Solr schema via an HTTP-API call, targeting your Solr instance:
-By default, it will download from your Dataverse installation at `http://localhost:8080` and reload Solr at `http://localhost:8983`.
-You may use the following environment variables with this script or mix'n'match with options:
+``curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"``
-==================== ====== =============================================== =========================================================
-Environment variable Option Description Example
-==================== ====== =============================================== =========================================================
-`DATAVERSE_URL` `-d` Provide the URL to your Dataverse installation *http://localhost:8080*
-`SOLR_URL` `-s` Provide the URL to your Solr instance *http://localhost:8983*
-`UNBLOCK_KEY` `-u` If your installation has a blocked admin API *xyz* or */secrets/unblock.key*
- endpoint, you can provide either the key itself
- or a path to a keyfile
-`TARGET` `-t` Provide the config directory of your Solr core */usr/local/solr/solr-8.8.1/server/solr/collection1/conf*
- "collection1"
-==================== ====== =============================================== =========================================================
+You can easily roll your own little script to automate the process (which might involve fetching the schema bits
+from some place else than your Dataverse installation).
-See the :doc:`/installation/prerequisites/` section of the Installation Guide for a suggested location on disk for the Solr schema file.
+Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate
+a necessary re-index, but for your custom metadata you will need to keep track on your own.
-Please note that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields.
+Please note also that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have
+added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you
+don't care about) to create a complete list of fields. (This might change in the future.)
Reloading a Metadata Block
--------------------------
diff --git a/scripts/api/setup-optional-harvard.sh b/scripts/api/setup-optional-harvard.sh
index e51846bd9e0..0cf6ea4d393 100755
--- a/scripts/api/setup-optional-harvard.sh
+++ b/scripts/api/setup-optional-harvard.sh
@@ -52,5 +52,5 @@ curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @da
curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/customCHIA.tsv -H "Content-type: text/tab-separated-values"
curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/customDigaai.tsv -H "Content-type: text/tab-separated-values"
curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/custom_hbgdki.tsv -H "Content-type: text/tab-separated-values"
-echo "Because you have loaded custom metadata blocks, you need to update the include files pulled in by Solr's schema.xml. On the Solr server, you can try running the updateSchemaMDB.sh script mentioned in the Metadata Customization section of the Admin Guide or follow the manual steps listed there."
+echo "Because you have loaded custom metadata blocks, you need to update the include files pulled in by Solr's schema.xml. On the Solr server, you can try running the update-fields.sh script mentioned in the Metadata Customization section of the Admin Guide or follow the manual steps listed there."
echo
diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile
index fe26bb5d6c6..006b5c818cd 100644
--- a/scripts/installer/Makefile
+++ b/scripts/installer/Makefile
@@ -4,7 +4,7 @@ GLASSFISH_SETUP_SCRIPT=${INSTALLER_ZIP_DIR}/as-setup.sh
API_SCRIPTS=${INSTALLER_ZIP_DIR}/setup-datasetfields.sh ${INSTALLER_ZIP_DIR}/setup-users.sh ${INSTALLER_ZIP_DIR}/setup-builtin-roles.sh ${INSTALLER_ZIP_DIR}/setup-dvs.sh ${INSTALLER_ZIP_DIR}/data ${INSTALLER_ZIP_DIR}/setup-identity-providers.sh ${INSTALLER_ZIP_DIR}/setup-all.sh ${INSTALLER_ZIP_DIR}/post-install-api-block.sh
JHOVE_CONFIG=${INSTALLER_ZIP_DIR}/jhove.conf
JHOVE_SCHEMA=${INSTALLER_ZIP_DIR}/jhoveConfig.xsd
-SOLR_SCHEMA=${INSTALLER_ZIP_DIR}/schema.xml ${INSTALLER_ZIP_DIR}/schema_dv_mdb_fields.xml ${INSTALLER_ZIP_DIR}/updateSchemaMDB.sh
+SOLR_SCHEMA=${INSTALLER_ZIP_DIR}/schema.xml ${INSTALLER_ZIP_DIR}/update-fields.sh
SOLR_CONFIG=${INSTALLER_ZIP_DIR}/solrconfig.xml
PYTHON_FILES=${INSTALLER_ZIP_DIR}/README_python.txt ${INSTALLER_ZIP_DIR}/installConfig.py ${INSTALLER_ZIP_DIR}/installUtils.py ${INSTALLER_ZIP_DIR}/install.py ${INSTALLER_ZIP_DIR}/installAppServer.py ${INSTALLER_ZIP_DIR}/requirements.txt ${INSTALLER_ZIP_DIR}/default.config ${INSTALLER_ZIP_DIR}/interactive.config
INSTALL_SCRIPT=${INSTALLER_ZIP_DIR}/install
@@ -56,9 +56,9 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
@echo copying jhove schema file
/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
-${SOLR_SCHEMA}: ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/schema_dv_mdb_fields.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
+${SOLR_SCHEMA}: ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR}
@echo copying Solr schema file
- /bin/cp ../../conf/solr/8.8.1/schema*.xml ../../conf/solr/8.8.1/updateSchemaMDB.sh ${INSTALLER_ZIP_DIR}
+ /bin/cp ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR}
${SOLR_CONFIG}: ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
@echo copying Solr config file
diff --git a/tests/shell/.shellspec b/tests/shell/.shellspec
new file mode 100644
index 00000000000..c99d2e7396e
--- /dev/null
+++ b/tests/shell/.shellspec
@@ -0,0 +1 @@
+--require spec_helper
diff --git a/tests/shell/data/solr/chain-output.xml b/tests/shell/data/solr/chain-output.xml
new file mode 100644
index 00000000000..a54414ad244
--- /dev/null
+++ b/tests/shell/data/solr/chain-output.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
diff --git a/tests/shell/data/solr/empty-source.xml b/tests/shell/data/solr/empty-source.xml
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/shell/data/solr/invalid-source.xml b/tests/shell/data/solr/invalid-source.xml
new file mode 100644
index 00000000000..49b7a7c7798
--- /dev/null
+++ b/tests/shell/data/solr/invalid-source.xml
@@ -0,0 +1,4 @@
+Bla bla bla
+Anything
+Foobar
+But no field or copyField
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-missing-copyfield-end.xml b/tests/shell/data/solr/mark-missing-copyfield-end.xml
new file mode 100644
index 00000000000..3ac2791f10d
--- /dev/null
+++ b/tests/shell/data/solr/mark-missing-copyfield-end.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-missing-copyfield-start.xml b/tests/shell/data/solr/mark-missing-copyfield-start.xml
new file mode 100644
index 00000000000..547b9c5bbcb
--- /dev/null
+++ b/tests/shell/data/solr/mark-missing-copyfield-start.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-missing-field-end.xml b/tests/shell/data/solr/mark-missing-field-end.xml
new file mode 100644
index 00000000000..eea4918440d
--- /dev/null
+++ b/tests/shell/data/solr/mark-missing-field-end.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-missing-field-start.xml b/tests/shell/data/solr/mark-missing-field-start.xml
new file mode 100644
index 00000000000..082a11fbddc
--- /dev/null
+++ b/tests/shell/data/solr/mark-missing-field-start.xml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-nolinebreak.xml b/tests/shell/data/solr/mark-nolinebreak.xml
new file mode 100644
index 00000000000..a9aa476008c
--- /dev/null
+++ b/tests/shell/data/solr/mark-nolinebreak.xml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-nonunique-copyfield-end.xml b/tests/shell/data/solr/mark-nonunique-copyfield-end.xml
new file mode 100644
index 00000000000..d8ea99842c3
--- /dev/null
+++ b/tests/shell/data/solr/mark-nonunique-copyfield-end.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-nonunique-copyfield-start.xml b/tests/shell/data/solr/mark-nonunique-copyfield-start.xml
new file mode 100644
index 00000000000..fd7636ec80f
--- /dev/null
+++ b/tests/shell/data/solr/mark-nonunique-copyfield-start.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-nonunique-field-end.xml b/tests/shell/data/solr/mark-nonunique-field-end.xml
new file mode 100644
index 00000000000..da03ef65e21
--- /dev/null
+++ b/tests/shell/data/solr/mark-nonunique-field-end.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-nonunique-field-start.xml b/tests/shell/data/solr/mark-nonunique-field-start.xml
new file mode 100644
index 00000000000..1a441cdf521
--- /dev/null
+++ b/tests/shell/data/solr/mark-nonunique-field-start.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-order-1.xml b/tests/shell/data/solr/mark-order-1.xml
new file mode 100644
index 00000000000..635143fe2f0
--- /dev/null
+++ b/tests/shell/data/solr/mark-order-1.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-order-2.xml b/tests/shell/data/solr/mark-order-2.xml
new file mode 100644
index 00000000000..7b085ca1fea
--- /dev/null
+++ b/tests/shell/data/solr/mark-order-2.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/mark-order-3.xml b/tests/shell/data/solr/mark-order-3.xml
new file mode 100644
index 00000000000..a8b13b144e5
--- /dev/null
+++ b/tests/shell/data/solr/mark-order-3.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/shell/data/solr/minimal-schema.xml b/tests/shell/data/solr/minimal-schema.xml
new file mode 100644
index 00000000000..004e96ff604
--- /dev/null
+++ b/tests/shell/data/solr/minimal-schema.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/shell/data/solr/minimal-source.xml b/tests/shell/data/solr/minimal-source.xml
new file mode 100644
index 00000000000..a6f95caa294
--- /dev/null
+++ b/tests/shell/data/solr/minimal-source.xml
@@ -0,0 +1,2 @@
+
+
\ No newline at end of file
diff --git a/tests/shell/spec/spec_helper.sh b/tests/shell/spec/spec_helper.sh
new file mode 100644
index 00000000000..93f19083cd2
--- /dev/null
+++ b/tests/shell/spec/spec_helper.sh
@@ -0,0 +1,24 @@
+# shellcheck shell=sh
+
+# Defining variables and functions here will affect all specfiles.
+# Change shell options inside a function may cause different behavior,
+# so it is better to set them here.
+# set -eu
+
+# This callback function will be invoked only once before loading specfiles.
+spec_helper_precheck() {
+ # Available functions: info, warn, error, abort, setenv, unsetenv
+ # Available variables: VERSION, SHELL_TYPE, SHELL_VERSION
+ : minimum_version "0.28.1"
+}
+
+# This callback function will be invoked after a specfile has been loaded.
+spec_helper_loaded() {
+ :
+}
+
+# This callback function will be invoked after core modules has been loaded.
+spec_helper_configure() {
+ # Available functions: import, before_each, after_each, before_all, after_all
+ : import 'support/custom_matcher'
+}
diff --git a/tests/shell/spec/update_fields_spec.sh b/tests/shell/spec/update_fields_spec.sh
new file mode 100644
index 00000000000..829b67fdcc4
--- /dev/null
+++ b/tests/shell/spec/update_fields_spec.sh
@@ -0,0 +1,169 @@
+#shellcheck shell=sh
+
+update_fields() {
+ ../../conf/solr/8.8.1/update-fields.sh "$@"
+}
+
+Describe "Update fields command"
+
+ Describe "can operate on upstream data"
+ copyUpstreamSchema() { cp ../../conf/solr/8.8.1/schema.xml data/solr/upstream-schema.xml; }
+ AfterAll 'copyUpstreamSchema'
+
+ Path schema-xml="../../conf/solr/8.8.1/schema.xml"
+ It "needs upstream schema.xml"
+ The path schema-xml should be exist
+ End
+ End
+
+ Describe "schema.xml validation"
+ It "throws error when no schema.xml target given"
+ When run update_fields
+ The status should equal 2
+ The error should include "Cannot find or write"
+ End
+
+ Describe "throws error when missing mark"
+ Parameters
+ "#1" "SCHEMA-FIELDS::BEGIN" data/solr/mark-missing-field-start.xml
+ "#2" "SCHEMA-FIELDS::END" data/solr/mark-missing-field-end.xml
+ "#3" "SCHEMA-COPY-FIELDS::BEGIN" data/solr/mark-missing-copyfield-start.xml
+ "#4" "SCHEMA-COPY-FIELDS::END" data/solr/mark-missing-copyfield-end.xml
+ End
+
+ It "$2"
+ When run update_fields "$3"
+ The status should equal 2
+ The error should include "$2"
+ End
+ End
+
+ Describe "throws error when non-unique mark"
+ Parameters
+ "#1" "SCHEMA-FIELDS::BEGIN" data/solr/mark-nonunique-field-start.xml
+ "#2" "SCHEMA-FIELDS::END" data/solr/mark-nonunique-field-end.xml
+ "#3" "SCHEMA-COPY-FIELDS::BEGIN" data/solr/mark-nonunique-copyfield-start.xml
+ "#4" "SCHEMA-COPY-FIELDS::END" data/solr/mark-nonunique-copyfield-end.xml
+ End
+
+ It "$2"
+ When run update_fields "$3"
+ The status should equal 2
+ The error should include "guards are not unique"
+ End
+ End
+
+ Describe "throws error when marks not in correct order"
+ Parameters
+ "#1" data/solr/mark-order-1.xml
+ "#2" data/solr/mark-order-2.xml
+ "#3" data/solr/mark-order-3.xml
+ End
+
+ It "$1"
+ When run update_fields "$2"
+ The status should equal 2
+ The error should include "guards are not in correct order"
+ End
+ End
+
+ It "throws error when marks not in exclusive line"
+ When run update_fields data/solr/mark-nolinebreak.xml
+ The status should equal 2
+ The error should include "is not on an exclusive line"
+ End
+ End
+
+ Describe "reading input"
+ Describe "fails because"
+ # Test if $CI is set (always true inside Github Workflow)
+ detect_github_action() { ! test -z ${CI:+x}; }
+
+ It "throws error when no source given"
+ Skip if "running on Github Action" detect_github_action
+ When run update_fields data/solr/minimal-schema.xml
+ The status should equal 2
+ The error should include "provide source file or piped input"
+ End
+
+ It "throws error when given file not found"
+ When run update_fields data/solr/minimal-schema.xml foobar
+ The status should equal 2
+ The error should include "Cannot read"
+ The error should include "foobar"
+ End
+
+ It "throws error when given file is empty"
+ When run update_fields data/solr/minimal-schema.xml data/solr/empty-source.xml
+ The status should equal 2
+ The error should include "Cannot read"
+ The error should include "empty-source.xml"
+ End
+
+ It "throws error when given invalid source file"
+ When run update_fields data/solr/minimal-schema.xml data/solr/invalid-source.xml
+ The status should equal 2
+ The error should include "No or "
+ End
+
+ It "throws error when given invalid stdin source"
+ Data < data/solr/invalid-source.xml
+ When run update_fields data/solr/minimal-schema.xml
+ The status should equal 2
+ The error should include "No or "
+ End
+ End
+
+ Describe "succeeds because"
+ setup() { cp data/solr/minimal-schema.xml data/solr/minimal-schema-work.xml; }
+ cleanup() { rm data/solr/minimal-schema-work.xml; }
+ BeforeEach 'setup'
+ AfterEach 'cleanup'
+
+ deleteUpstreamSchema() { rm data/solr/upstream-schema.xml; }
+ AfterAll 'deleteUpstreamSchema'
+
+ match_content() {
+ grep -q "$@" "${match_content}"
+ }
+
+ It "prints nothing when editing minimal schema"
+ Data < data/solr/minimal-source.xml
+ When run update_fields data/solr/minimal-schema-work.xml
+ The status should equal 0
+ The output should equal ""
+ The path data/solr/minimal-schema-work.xml should be file
+ The path data/solr/minimal-schema-work.xml should satisfy match_content "