7662 make solrconfig less static

doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service

@@ -8,6 +8,7 @@ Type = forking
 WorkingDirectory = /usr/local/solr/solr-8.11.1
 ExecStart = /usr/local/solr/solr-8.11.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
 ExecStop = /usr/local/solr/solr-8.11.1/bin/solr stop
+Environment="SOLR_OPTS=-Dsolr.jetty.request.header.size=102400"
 LimitNOFILE=65000
 LimitNPROC=65000
 Restart=on-failure

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -516,7 +516,7 @@ the Solr schema configuration, including any enabled metadata schemas:
 
 ``curl "http://localhost:8080/api/admin/index/solr/schema"``
 
-You can use :download:`update-fields.sh <../../../../conf/solr/8.11.1/update-fields.sh>` to easily add these to the
+You can use :download:`update-fields.sh <../../../../modules/solr-configset/src/main/scripts/update-fields.sh>` to easily add these to the
 Solr schema you installed for your Dataverse installation.
 
 The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of

doc/sphinx-guides/source/conf.py

@@ -18,6 +18,9 @@
 sys.path.insert(0, os.path.abspath('../../'))
 import sphinx_bootstrap_theme
 
+import xml.etree.ElementTree as et
+pom = et.parse("../../../modules/dataverse-parent/pom.xml")
+ns = {"mvn": "http://maven.apache.org/POM/4.0.0"}
 
 # Activate the theme.
 # html_theme = 'bootstrap'
@@ -438,4 +441,5 @@
 rst_prolog = """
 .. |toctitle| replace:: Contents:
 .. |anotherSub| replace:: Yes, there can be multiple.
-"""
+.. |solr_version| replace:: {solr_version}
+""".format(solr_version=pom.find("./mvn:properties/mvn:solr.version", ns).text)

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -154,86 +154,164 @@ Configuring Database Access for the Dataverse Installation (and the Dataverse So
 Solr
 ----
 
-The Dataverse Software search index is powered by Solr.
+The Dataverse Software search index is powered by `Solr <https://solr.apache.org>`_.
 
 Supported Versions
 ==================
 
-The Dataverse Software has been tested with Solr version 8.11.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
+The Dataverse Software has been tested with Solr version |solr_version|. Future releases in the 8.x series are likely to
+be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x)
+are not supported.
+
+- Releases up to 4.20 supported Solr 7.x.x.
+- Releases 5.0 to 5.3 supported Solr 7.7.2.
+- Releases 5.4 to 5.9 supported Solr 8.8.1.
+- Releases since 5.10 support Solr |solr_version|
+
 
 Installing Solr
 ===============
 
-You should not run Solr as root. Create a user called ``solr`` and a directory to install Solr into::
+Note: this guide describes setting up a small installation, using the Solr *standalone* mode. For larger installations or
+higher availability requirements, please take a look at `Solr Cloud <https://solr.apache.org/guide/solrcloud.html>`_ mode.
+
+Optional Step 0:
 
-        useradd solr
+- Solr launches asynchronously and attempts to use the ``lsof`` binary to watch for its own availability.
+  Installation of this package isn't required but will prevent a warning in the log at startup. (Use ``dnf``, ``yum`` or
+  ``apt-get`` to install this standard package.)
+- Solr 8.x runs on Java 11 (same as your Dataverse installation). Remember to install it when running Solr on a
+  separated machine.
+
+**Step 1**: You should **not** run Solr as ``root``! Create a user and group called ``solr`` (as root) and a directory to
+install Solr into:
+
+.. parsed-literal::
         mkdir /usr/local/solr
-        chown solr:solr /usr/local/solr
+        groupadd -r --gid 8983 solr
+        useradd -r --home-dir /usr/local/solr --uid 8983 --gid 8983 solr
+        chown solr: /usr/local/solr
 
-Become the ``solr`` user and then download and configure Solr::
+**Step 2:** Become the ``solr`` user and then download and configure Solr:
 
-        su - solr
+.. parsed-literal::
+        sudo -u solr -s
         cd /usr/local/solr
-        wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz
-        tar xvzf solr-8.11.1.tgz
-        cd solr-8.11.1
-        cp -r server/solr/configsets/_default server/solr/collection1
+        wget https://archive.apache.org/dist/lucene/solr/|solr_version|/solr-|solr_version|.tgz
+        tar xvzf solr-|solr_version|.tgz
+        exit
 
-You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::
 
-        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
-        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
+Solr Init Script
+================
 
-Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ``<str name="qf">`` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev .
+**Step 3:** Once you installed Solr, you need to add to the init system to start on boot, stop on shutdown etc. Please choose the
+right option for your underlying Linux operating system. *It will not be necessary to execute both!*
 
-A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.11.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
+SystemD based systems
+^^^^^^^^^^^^^^^^^^^^^
 
-Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``::
+For systems running systemd (like RedHat or derivatives since 7, Debian since 9, Ubuntu since 15.04), as root, download
+:download:`solr.service<../_static/installation/files/etc/systemd/solr.service>` and place it in ``/tmp``. Then start
+Solr and configure it to start at boot with the following commands (run as root again):
 
-        solr soft nproc 65000
-        solr hard nproc 65000
-        solr soft nofile 65000
-        solr hard nofile 65000
+.. parsed-literal::
+        cp /tmp/solr.service /etc/systemd/system
+        systemctl daemon-reload
+        systemctl start solr.service
+        systemctl enable solr.service
 
-On operating systems which use systemd such as RHEL/derivative, you may then add a line like LimitNOFILE=65000 for the number of open file descriptors and a line with LimitNPROC=65000 for the max processes to the systemd unit file, or adjust the limits on a running process using the prlimit tool::
+SysVinit based systems
+^^^^^^^^^^^^^^^^^^^^^^
 
-        # sudo prlimit --pid pid --nofile=65000:65000
+For (older) systems using init.d (like CentOS 6 or Devuan), download this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>`
+and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands (run as root again):
 
-Solr launches asynchronously and attempts to use the ``lsof`` binary to watch for its own availability. Installation of this package isn't required but will prevent a warning in the log at startup::
+.. parsed-literal::
+        cp /tmp/solr /etc/init.d
+        service start solr
+        chkconfig solr on
 
-	# yum install lsof
 
-Finally, you need to tell Solr to create the core "collection1" on startup::
+Creating Solr Core
+==================
 
-        echo "name=collection1" > /usr/local/solr/solr-8.11.1/server/solr/collection1/core.properties
+Solr Cores hold the actual data of your index. They get created from templates called "config sets". We provide a
+template that has been tuned carefully for usage within a Dataverse installation and is distributed as a ZIP file.
 
-Solr Init Script
-================
+Note: The Dataverse Project team has customized the cores ``solrconfig.xml`` to boost Solr search results that come from
+certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections
+over Datasets. If you would like to remove this, edit this file and remove the ``<str name="qf">`` element and its
+contents. If you have ideas about how this boosting could be improved, feel free to contact us through our
+`Google Group <https://groups.google.com/forum/#!forum/dataverse-dev>`_.
 
-Please choose the right option for your underlying Linux operating system.
-It will not be necessary to execute both!
+**Step 4:** If not already done, please download the latest release package ``dvinstall.zip`` at
+https://github.com/IQSS/dataverse/releases.
 
-For systems running systemd (like RedHat or derivatives since 7, Debian since 9, Ubuntu since 15.04), as root, download :download:`solr.service<../_static/installation/files/etc/systemd/solr.service>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
+**Step 5:** Extract our Solr Dataverse config set from it and unpack the configset directory:
 
-        cp /tmp/solr.service /etc/systemd/system
-        systemctl daemon-reload
-        systemctl start solr.service
-        systemctl enable solr.service
+.. parsed-literal::
+        sudo -u solr -s
+        cd solr-|solr_version|/server/solr/configsets
+        unzip path/to/dvinstall.zip solr-configset.zip
+        unzip solr-configset.zip
 
-For systems using init.d (like CentOS 6), download this :download:`Solr init script <../_static/installation/files/etc/init.d/solr>` and place it in ``/tmp``. Then start Solr and configure it to start at boot with the following commands::
+**Step 6:** Create the core within your running Solr instance:
 
-        cp /tmp/solr /etc/init.d
-        service start solr
-        chkconfig solr on
+.. parsed-literal::
+        /usr/local/solr/solr-|solr_version|/bin/solr create -c collection1 -d dataverse
+
+
+Tuning Solr
+===========
+
+The next steps are mostly extracted from the recommendations for
+`"Taking Solr to Production" <https://solr.apache.org/guide/taking-solr-to-production.html>`_.
+
+They are mostly necessary for older Linux distributions using System V init systems. If you are using our
+SystemD unit file (see above), they may be skipped.
+
+1. A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr.
+   Edit ``/usr/local/solr/*/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``.
+
+   Alternative: use ``SOLR_OPTS`` to set the system property (see Solr docs linked above).
+
+2. Solr will warn about needing to increase the number of file descriptors and max processes in a production environment
+   but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000
+   to the init script, and the following to ``/etc/security/limits.conf``:
+
+   .. parsed-literal::
+        solr soft nproc 65000
+        solr hard nproc 65000
+        solr soft nofile 65000
+        solr hard nofile 65000
+
+   Note: This is not necessary with SystemD, which ignores these settings (see unit file instead)!
+   If not using our unit file, you may need to add a line like ``LimitNOFILE=65000`` for the number of open file
+   descriptors and a line with ``LimitNPROC=65000`` for the max processes to the systemd unit file.
+
+   Alternative: adjust the limits on a running process using the ``prlimit`` tool:
+
+   .. parsed-literal::
+         sudo prlimit --pid pid --nofile=65000:65000
 
 Securing Solr
 =============
 
-Our sample init script and systemd service file linked above tell Solr to only listen on localhost (127.0.0.1). We strongly recommend that you also use a firewall to block access to the Solr port (8983) from outside networks, for added redundancy.
+Our sample init script and systemd service file linked above tell Solr to only listen on localhost (127.0.0.1). We
+strongly recommend that you also use a firewall to block access to the Solr port (8983) from outside networks, for
+added redundancy.
 
-It is **very important** not to allow direct access to the Solr API from outside networks! Otherwise, any host that can reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://lucene.apache.org/solr/guide/7_3/securing-solr.html. A particularly serious security issue that has been identified recently allows a potential intruder to remotely execute arbitrary code on the system. See `RCE in Solr via Velocity Template <https://github.com/veracode-research/solr-injection#7-cve-2019-xxxx-rce-via-velocity-template-by-_s00py>`_ for more information.
+It is **very important** not to allow direct access to the Solr API from outside networks! Otherwise, any host that can
+reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For
+more information, please see https://lucene.apache.org/solr/guide/7_3/securing-solr.html. A particularly serious
+security issue that has been identified recently allows a potential intruder to remotely execute arbitrary code on the
+system. See `RCE in Solr via Velocity Template <https://github.com/veracode-research/solr-injection#7-cve-2019-xxxx-rce-via-velocity-template-by-_s00py>`_
+for more information.
 
-If you're running your Dataverse installation across multiple service hosts you'll want to remove the jetty.host argument (``-j jetty.host=127.0.0.1``) from the startup command line, but make sure Solr is behind a firewall and only accessible by the Dataverse installation host(s), by specific ip address(es).
+If you're running your Dataverse installation across multiple service hosts you'll want to remove the jetty.host
+argument (``-j jetty.host=127.0.0.1``) from the startup command line, but make sure Solr is behind a firewall and only
+accessible by the Dataverse installation host(s), by specific ip address(es).
 
 We additionally recommend that the Solr service account's shell be disabled, as it isn't necessary for daily operation::
 
@@ -247,7 +325,10 @@ or simply prepend each command you would run as the Solr user with "sudo -u solr
 
         # sudo -u solr command
 
-Finally, we would like to reiterate that it is simply never a good idea to run Solr as root! Running the process as a non-privileged user would substantially minimize any potential damage even in the event that the instance is compromised.
+Finally, we would like to reiterate that it is simply never a good idea to run Solr as root! Running the process as
+non-privileged user would substantially minimize any potential damage even in the event that the instance is compromised.
+
+
 
 jq
 --

modules/dataverse-parent/pom.xml

@@ -15,6 +15,7 @@
         <module>../../scripts/zipdownload</module>
         <module>../container-base</module>
         <module>../dataverse-spi</module>
+        <module>../solr-configset</module>
     </modules>
 
     <!-- Transitive dependencies, bigger library "bill of materials" (BOM) and
@@ -53,6 +54,11 @@
                 <artifactId>postgresql</artifactId>
                 <version>${postgresql.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.solr</groupId>
+                <artifactId>solr-solrj</artifactId>
+                <version>${solr.version}</version>
+            </dependency>
             <dependency>
                 <groupId>commons-logging</groupId>
                 <artifactId>commons-logging</artifactId>
@@ -126,6 +132,12 @@
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>
+
+            <dependency>
+                <groupId>org.junit.jupiter</groupId>
+                <artifactId>junit-jupiter</artifactId>
+                <version>${junit.jupiter.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -200,6 +212,8 @@
 
         <!-- Container related -->
         <fabric8-dmp.version>0.43.0</fabric8-dmp.version>
+        <solr.collection>collection1</solr.collection>
+        <solr.configset>dataverse</solr.configset> <!-- Note: container usage only for now, might be reused in code -->
     </properties>
 
     <pluginRepositories>
@@ -407,7 +421,6 @@
                 -->
                 <!-- <payara.version>5.2022.5</payara.version> -->
             </properties>
-
             <build>
                 <plugins>
                     <!-- This will get the current commit id to include in image tags of container builds as Maven properties -->
@@ -433,8 +446,6 @@
                     </plugin>
                 </plugins>
             </build>
-
         </profile>
     </profiles>
-
 </project>