From de0d244c5e89d2c58b0718194cac8ffd57b0f4ef Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Thu, 31 Aug 2023 10:01:21 -0400
Subject: [PATCH 01/14] test user managed index replication

---
 solr/bin/solr                                 |   1 +
 .../org/apache/solr/cli/RunExampleTool.java   |  17 ++-
 .../solr/cli/TestSolrCLIRunExample.java       |   5 +-
 solr/packaging/test/test_replication.bats     | 125 ++++++++++++++++++
 4 files changed, 145 insertions(+), 3 deletions(-)
 create mode 100644 solr/packaging/test/test_replication.bats

diff --git a/solr/bin/solr b/solr/bin/solr
index 8a6d05607570..a5073f32f601 100644
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1263,6 +1263,7 @@ if [ $# -gt 0 ]; then
         ;;
         -t|-data.home)
             SOLR_DATA_HOME="$2"
+            PASS_TO_RUN_EXAMPLE+=("-t" "$SOLR_DATA_HOME")
             shift 2
         ;;
         -e|-example)
diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index 80dcdb41c72b..6294bd773757 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -118,6 +118,13 @@ public List<Option> getOptions() {
             .desc("Path to the Solr server directory.")
             .longOpt("serverDir")
             .build(),
+        Option.builder("t")
+                .argName("DIR")
+                .hasArg()
+                .required(false)
+                .desc("Path to the Solr data home directory.")
+                .longOpt("dataHomeDir")
+                .build(),
         Option.builder("force")
             .argName("FORCE")
             .desc("Force option in case Solr is run as root.")
@@ -591,6 +598,9 @@ protected Map<String, Object> startSolr(
     String forceArg = cli.hasOption("force") ? " -force" : "";
     String verboseArg = verbose ? "-V" : "";
 
+    String dataDirArg = cli.hasOption("dataHomeDir") ? "-t" : "";
+    String dataDirValue = cli.hasOption("dataHomeDir") ? cli.getOptionValue("dataHomeDir") : "";
+
     String addlOpts = cli.getOptionValue('a');
     String addlOptsArg = (addlOpts != null) ? " -a \"" + addlOpts + "\"" : "";
 
@@ -610,7 +620,7 @@ protected Map<String, Object> startSolr(
     String startCmd =
         String.format(
             Locale.ROOT,
-            "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s %s",
+            "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s %s %s %s",
             callScript,
             cloudModeArg,
             port,
@@ -620,6 +630,8 @@ protected Map<String, Object> startSolr(
             memArg,
             forceArg,
             verboseArg,
+                dataDirArg,
+                dataDirValue,
             extraArgs,
             addlOptsArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
@@ -627,6 +639,9 @@ protected Map<String, Object> startSolr(
     echo("\nStarting up Solr on port " + port + " using command:");
     echo(startCmd + "\n");
 
+    System.out.println("\nStarting up Solr on port " + port + " using command:");
+    System.out.println(startCmd + "\n");
+
     String solrUrl =
         String.format(
             Locale.ROOT, "%s://%s:%d/solr", urlScheme, (host != null ? host : "localhost"), port);
diff --git a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
index d6f7da99eee4..0a6af3d34a24 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
@@ -316,7 +316,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  @LuceneTestCase.Nightly
+  //@LuceneTestCase.Nightly
   public void testTechproductsExample() throws Exception {
     testExample("techproducts");
   }
@@ -356,7 +356,8 @@ protected void testExample(String exampleName) throws Exception {
             "-e", exampleName,
             "-serverDir", solrServerDir.getAbsolutePath(),
             "-exampleDir", solrExampleDir.getAbsolutePath(),
-            "-p", String.valueOf(bindPort)
+            "-p", String.valueOf(bindPort),
+                  "-t","dude"
           };
 
       // capture tool output to stdout
diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
new file mode 100644
index 000000000000..a4472f1edcdd
--- /dev/null
+++ b/solr/packaging/test/test_replication.bats
@@ -0,0 +1,125 @@
+#!/usr/bin/env bats
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load bats_helper
+
+setup() {
+  common_clean_setup
+}
+
+teardown() {
+  # save a snapshot of SOLR_HOME for failed tests
+  save_home_on_failure
+
+  solr stop -all >/dev/null 2>&1
+}
+
+@test "user managed index replication without zookeeper" {
+  skip
+
+  # This test is fragile.  I think we are creating files etc in solr/packaging/build/solr-10.0.0-SNAPSHOT/example/techproducts
+  # which means you may need to run gradle clean between runs or we can't start up Solr.
+  # I think we can't run both bats test either, one after the other ;-( 
+  
+  mkdir -p test_data_dir_8983
+  mkdir -p test_data_dir_7574
+  
+  solr start -e techproducts -p 8983 -t test_data_dir_8983 -V 
+  solr start -e techproducts -p 7574 -t test_data_dir_7574 -V -Dsolr.disable.allowUrls=true
+  solr assert --started http://localhost:8983 --timeout 5000
+  solr assert --started http://localhost:7574 --timeout 5000
+  
+  run curl 'http://localhost:8983/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":32'
+  
+  # Not totally sure why this didn't load it's data, but it works for our needs!
+  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":0'  
+  
+  run curl 'http://localhost:8983/solr/techproducts/replication?command=enablereplication'
+  assert_output --partial '"OK"'
+  run curl 'http://localhost:7574/solr/techproducts/replication?command=fetchindex&leaderUrl=http://localhost:8983/solr/techproducts'
+  assert_output --partial '"OK"'
+  
+  # Wish we could block on fetchindex..   Does checking details help?
+  sleep 5
+  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":32' 
+  
+  run bash -c 'solr stop -all 2>&1'
+  refute_output --partial 'forcefully killing'
+}
+
+@test "user managed index replication WITH Zookeeper" {
+  # This test is fragile.  I think we are creating files etc in solr/packaging/build/solr-10.0.0-SNAPSHOT/example/techproducts
+  # which means you may need to run gradle clean between runs or we can't start up Solr.
+  # I think we can't run both bats test either, one after the other ;-( 
+  
+  mkdir -p test_data_dir_8983
+  
+  solr start -c -e techproducts -p 8983 -t test_data_dir_8983 -V
+  solr start -c -p 7574 -V -Dsolr.disable.allowUrls=true
+  solr assert --started http://localhost:8983 --timeout 5000
+  solr assert --started http://localhost:7574 --timeout 5000
+  
+  run curl 'http://localhost:8983/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":32'
+  run curl 'http://localhost:8983/solr/techproducts_shard1_replica_n1/select?q=*:*'
+  assert_output --partial '"numFound":32'
+  
+  # setup the follower
+  local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
+  test -d $source_configset_dir
+
+  run solr zk upconfig -d ${source_configset_dir} -n techproducts -z localhost:8574
+  assert_output --partial "Uploading"
+  run solr create -c techproducts -d sample_techproducts_configs -solrUrl http://localhost:7574
+  assert_output --partial "Created collection 'techproducts'"
+    
+  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":0'  
+  
+  run curl -X POST -H 'Content-type:application/json' -d '{"add-requesthandler": {"name": "/replication","class": "solr.ReplicationHandler","follower": { "leaderUrl": "http://localhost:8983/solr/techproducts_shard1_replica_n1" ,"pollInterval": "00:00:20" }}}' http://localhost:7574/api/collections/techproducts/config
+  assert_output --partial '"status":0' 
+  
+  sleep 30
+  
+  
+  run curl 'http://localhost:8983/solr/techproducts_shard1_replica_n1/replication?command=details'
+  assert_output --partial '"replicationEnabled":"true"'
+  #run curl 'http://localhost:7574/solr/techproducts_shard1_replica_n1/replication?command=fetchindex&leaderUrl=http://localhost:8983/solr/techproducts_shard1_replica_n1'
+  #assert_output --partial '"OK"'
+  
+  # Wish we could block on fetchindex..   Does checking details help?
+  sleep 5
+  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  assert_output --partial '"numFound":32' 
+  
+  # Create a doc on the leader and check the follower
+  run solr post -url http://localhost:8983/solr/techproducts/update -mode args -out -commit "{'add': {doc:{'id': 'newproduct'}}}"
+  assert_output --partial '"status":0'
+  
+  sleep 25
+  
+  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":33' 
+  
+  
+  
+  run bash -c 'solr stop -all 2>&1'
+  refute_output --partial 'forcefully killing'
+}

From 12649fb8e8b9b1f96a43e25fd3574dc4abc14dd1 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Thu, 31 Aug 2023 10:21:26 -0400
Subject: [PATCH 02/14] lint

---
 .../org/apache/solr/cli/RunExampleTool.java   | 19 ++++++++-----------
 .../solr/cli/TestSolrCLIRunExample.java       | 17 +++++++++++------
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index 6294bd773757..6e5906c4be01 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -119,12 +119,12 @@ public List<Option> getOptions() {
             .longOpt("serverDir")
             .build(),
         Option.builder("t")
-                .argName("DIR")
-                .hasArg()
-                .required(false)
-                .desc("Path to the Solr data home directory.")
-                .longOpt("dataHomeDir")
-                .build(),
+            .argName("DIR")
+            .hasArg()
+            .required(false)
+            .desc("Path to the Solr data home directory.")
+            .longOpt("dataHomeDir")
+            .build(),
         Option.builder("force")
             .argName("FORCE")
             .desc("Force option in case Solr is run as root.")
@@ -630,8 +630,8 @@ protected Map<String, Object> startSolr(
             memArg,
             forceArg,
             verboseArg,
-                dataDirArg,
-                dataDirValue,
+            dataDirArg,
+            dataDirValue,
             extraArgs,
             addlOptsArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
@@ -639,9 +639,6 @@ protected Map<String, Object> startSolr(
     echo("\nStarting up Solr on port " + port + " using command:");
     echo(startCmd + "\n");
 
-    System.out.println("\nStarting up Solr on port " + port + " using command:");
-    System.out.println(startCmd + "\n");
-
     String solrUrl =
         String.format(
             Locale.ROOT, "%s://%s:%d/solr", urlScheme, (host != null ? host : "localhost"), port);
diff --git a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
index 0a6af3d34a24..39aaf5e1c9a5 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
@@ -316,7 +316,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  //@LuceneTestCase.Nightly
+  // @LuceneTestCase.Nightly
   public void testTechproductsExample() throws Exception {
     testExample("techproducts");
   }
@@ -353,11 +353,16 @@ protected void testExample(String exampleName) throws Exception {
 
       String[] toolArgs =
           new String[] {
-            "-e", exampleName,
-            "-serverDir", solrServerDir.getAbsolutePath(),
-            "-exampleDir", solrExampleDir.getAbsolutePath(),
-            "-p", String.valueOf(bindPort),
-                  "-t","dude"
+            "-e",
+            exampleName,
+            "-serverDir",
+            solrServerDir.getAbsolutePath(),
+            "-exampleDir",
+            solrExampleDir.getAbsolutePath(),
+            "-p",
+            String.valueOf(bindPort),
+            "-t",
+            "dude"
           };
 
       // capture tool output to stdout

From 1f280513d9f51e4ac7debc52bac1b7bee18e85ec Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Thu, 31 Aug 2023 13:32:02 -0400
Subject: [PATCH 03/14] reduce the duration by polling more...  still not
 great.

---
 solr/packaging/test/test_replication.bats | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index a4472f1edcdd..6540ace3ffe0 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -93,10 +93,10 @@ teardown() {
   run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
   assert_output --partial '"numFound":0'  
   
-  run curl -X POST -H 'Content-type:application/json' -d '{"add-requesthandler": {"name": "/replication","class": "solr.ReplicationHandler","follower": { "leaderUrl": "http://localhost:8983/solr/techproducts_shard1_replica_n1" ,"pollInterval": "00:00:20" }}}' http://localhost:7574/api/collections/techproducts/config
+  run curl -X POST -H 'Content-type:application/json' -d '{"add-requesthandler": {"name": "/replication","class": "solr.ReplicationHandler","follower": { "leaderUrl": "http://localhost:8983/solr/techproducts_shard1_replica_n1" ,"pollInterval": "00:00:05" }}}' http://localhost:7574/api/collections/techproducts/config
   assert_output --partial '"status":0' 
   
-  sleep 30
+  sleep 10
   
   
   run curl 'http://localhost:8983/solr/techproducts_shard1_replica_n1/replication?command=details'
@@ -113,7 +113,7 @@ teardown() {
   run solr post -url http://localhost:8983/solr/techproducts/update -mode args -out -commit "{'add': {doc:{'id': 'newproduct'}}}"
   assert_output --partial '"status":0'
   
-  sleep 25
+  sleep 10
   
   run curl 'http://localhost:7574/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":33' 

From 59f6ae14e214ec5234fde2fffee224ce2f3cbca0 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Tue, 12 Sep 2023 08:16:26 -0400
Subject: [PATCH 04/14] back out the zookeeper POC

---
 solr/packaging/test/test_replication.bats | 63 +----------------------
 1 file changed, 1 insertion(+), 62 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 6540ace3ffe0..c16c319a5fc6 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -28,8 +28,7 @@ teardown() {
   solr stop -all >/dev/null 2>&1
 }
 
-@test "user managed index replication without zookeeper" {
-  skip
+@test "user managed index replication" {
 
   # This test is fragile.  I think we are creating files etc in solr/packaging/build/solr-10.0.0-SNAPSHOT/example/techproducts
   # which means you may need to run gradle clean between runs or we can't start up Solr.
@@ -60,66 +59,6 @@ teardown() {
   run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
   assert_output --partial '"numFound":32' 
   
-  run bash -c 'solr stop -all 2>&1'
-  refute_output --partial 'forcefully killing'
-}
-
-@test "user managed index replication WITH Zookeeper" {
-  # This test is fragile.  I think we are creating files etc in solr/packaging/build/solr-10.0.0-SNAPSHOT/example/techproducts
-  # which means you may need to run gradle clean between runs or we can't start up Solr.
-  # I think we can't run both bats test either, one after the other ;-( 
-  
-  mkdir -p test_data_dir_8983
-  
-  solr start -c -e techproducts -p 8983 -t test_data_dir_8983 -V
-  solr start -c -p 7574 -V -Dsolr.disable.allowUrls=true
-  solr assert --started http://localhost:8983 --timeout 5000
-  solr assert --started http://localhost:7574 --timeout 5000
-  
-  run curl 'http://localhost:8983/solr/techproducts/select?q=*:*'
-  assert_output --partial '"numFound":32'
-  run curl 'http://localhost:8983/solr/techproducts_shard1_replica_n1/select?q=*:*'
-  assert_output --partial '"numFound":32'
-  
-  # setup the follower
-  local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
-  test -d $source_configset_dir
-
-  run solr zk upconfig -d ${source_configset_dir} -n techproducts -z localhost:8574
-  assert_output --partial "Uploading"
-  run solr create -c techproducts -d sample_techproducts_configs -solrUrl http://localhost:7574
-  assert_output --partial "Created collection 'techproducts'"
-    
-  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
-  assert_output --partial '"numFound":0'  
-  
-  run curl -X POST -H 'Content-type:application/json' -d '{"add-requesthandler": {"name": "/replication","class": "solr.ReplicationHandler","follower": { "leaderUrl": "http://localhost:8983/solr/techproducts_shard1_replica_n1" ,"pollInterval": "00:00:05" }}}' http://localhost:7574/api/collections/techproducts/config
-  assert_output --partial '"status":0' 
-  
-  sleep 10
-  
-  
-  run curl 'http://localhost:8983/solr/techproducts_shard1_replica_n1/replication?command=details'
-  assert_output --partial '"replicationEnabled":"true"'
-  #run curl 'http://localhost:7574/solr/techproducts_shard1_replica_n1/replication?command=fetchindex&leaderUrl=http://localhost:8983/solr/techproducts_shard1_replica_n1'
-  #assert_output --partial '"OK"'
-  
-  # Wish we could block on fetchindex..   Does checking details help?
-  sleep 5
-  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
-  assert_output --partial '"numFound":32' 
-  
-  # Create a doc on the leader and check the follower
-  run solr post -url http://localhost:8983/solr/techproducts/update -mode args -out -commit "{'add': {doc:{'id': 'newproduct'}}}"
-  assert_output --partial '"status":0'
-  
-  sleep 10
-  
-  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*&rows=0'
-  assert_output --partial '"numFound":33' 
-  
-  
-  
   run bash -c 'solr stop -all 2>&1'
   refute_output --partial 'forcefully killing'
 }

From 76c0440b398b63649cc5161c4126175d022cde62 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 10:24:31 -0500
Subject: [PATCH 05/14] Point in time, but I have it all working locally
 manually, now time to batsify it

---
 solr/packaging/test/test_replication.bats | 101 +++++++++++++++++++---
 1 file changed, 87 insertions(+), 14 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index c16c319a5fc6..4f2d3918d4fe 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -34,31 +34,104 @@ teardown() {
   # which means you may need to run gradle clean between runs or we can't start up Solr.
   # I think we can't run both bats test either, one after the other ;-( 
   
-  mkdir -p test_data_dir_8983
-  mkdir -p test_data_dir_7574
+  # wish we had zkServerDataDir as a EnvUtil instead of system property!
   
-  solr start -e techproducts -p 8983 -t test_data_dir_8983 -V 
-  solr start -e techproducts -p 7574 -t test_data_dir_7574 -V -Dsolr.disable.allowUrls=true
-  solr assert --started http://localhost:8983 --timeout 5000
-  solr assert --started http://localhost:7574 --timeout 5000
+  mkdir -p cloud_5000
+  mkdir -p cloud_5100
+  mkdir -p cloud_5200
   
-  run curl 'http://localhost:8983/solr/techproducts/select?q=*:*'
+  # Get our three seperate independent Solr nodes running.
+  bin/solr start -f -c -p 5000 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5000 -DzkServerDataDir=$(pwd)/cloud_5000/zoo_data -v -V 
+  bin/solr start -f -c -p 5100 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5100 -DzkServerDataDir=$(pwd)/cloud_5100/zoo_data -v -V 
+  bin/solr start -f -c -p 5200 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5200 -DzkServerDataDir=$(pwd)/cloud_5200/zoo_data -v -V 
+  
+  solr assert --started http://localhost:5000 --timeout 5000
+  solr assert --started http://localhost:5100 --timeout 5000
+  solr assert --started http://localhost:5200 --timeout 5000
+  
+  solr assert -cloud http://localhost:5000
+  solr assert -cloud http://localhost:5100
+  solr assert -cloud http://localhost:5200  
+  
+  # Wish I loaded configset seperately...
+  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5000
+  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5100
+  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5200
+  
+  # Verify no data
+  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":0'
+  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":0'
+  run curl 'http://localhost:5200/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":0'  
+  
+  # Load XML formatted data into the leader
+  bin/solr post -type application/xml -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.xml
+  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":32'
   
-  # Not totally sure why this didn't load it's data, but it works for our needs!
-  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  # Confirm no replication
+  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":0'  
+  run curl 'http://localhost:5200/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":0'  
+  
+  # Setup the Leader for replication
+  run curl -X POST -H 'Content-type:application/json' -d '{
+    "add-requesthandler": {
+      "name": "/replication",
+      "class": "solr.ReplicationHandler",
+      "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
+      "maxNumberOfBackups":2            
+    }
+  }' "http://localhost:5000/solr/techproducts/config"
+  assert_output --partial '"status":0'
+  
+  run curl 'http://localhost:5000/solr/techproducts/replication?command=details'
+  assert_output --partial '"replicationEnabled":"true"'
+  
+  
+  # Setup the Repeater for replication
   
-  run curl 'http://localhost:8983/solr/techproducts/replication?command=enablereplication'
-  assert_output --partial '"OK"'
-  run curl 'http://localhost:7574/solr/techproducts/replication?command=fetchindex&leaderUrl=http://localhost:8983/solr/techproducts'
-  assert_output --partial '"OK"'
+  run curl -X POST -H 'Content-type:application/json' -d '{
+    "add-requesthandler": {
+      "name": "/replication",
+      "class": "solr.ReplicationHandler",
+      "follower":{ "leaderUrl": "http://localhost:5000/solr/techproducts/replication", "pollInterval":"00:00:02"},
+      "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
+      "maxNumberOfBackups":2            
+    }
+  }' "http://localhost:5100/solr/techproducts/config"
+  assert_output --partial '"status":0'
+  
+  run curl 'http://localhost:5100/solr/techproducts/replication?command=details'
+  assert_output --partial '"isPollingDisabled":"false"'
+
   
   # Wish we could block on fetchindex..   Does checking details help?
   sleep 5
-  run curl 'http://localhost:7574/solr/techproducts/select?q=*:*'
+  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":32' 
   
+  # Testing adding new data by adding JSON formatted data into the leader
+  bin/solr post -type application/json -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.json
+  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":36'
+  
+  sleep 5
+  
+  # Testing adding new data by adding CSV formatted data into the leader
+  bin/solr post -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.csv
+  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":46'
+  
+  sleep 5
+  
+  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":46'
+  
+  
   run bash -c 'solr stop -all 2>&1'
   refute_output --partial 'forcefully killing'
 }

From 28c7bc6b2de40234fa1828cf1f4f7d0d4a42207a Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 11:56:55 -0500
Subject: [PATCH 06/14] Bat enabled run!

---
 solr/packaging/test/test_replication.bats | 61 +++++++++++------------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 4f2d3918d4fe..56d6371448a2 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -28,22 +28,23 @@ teardown() {
   solr stop -all >/dev/null 2>&1
 }
 
-@test "user managed index replication" {
-
-  # This test is fragile.  I think we are creating files etc in solr/packaging/build/solr-10.0.0-SNAPSHOT/example/techproducts
-  # which means you may need to run gradle clean between runs or we can't start up Solr.
-  # I think we can't run both bats test either, one after the other ;-( 
+@test "user managed index replication with a twist" {
+  export SOLR_SECURITY_MANAGER_ENABLED=false
+  # This demonstrates traditional user managed cluster working as defined in
+  # https://solr.apache.org/guide/solr/latest/deployment-guide/cluster-types.html 
+  
+  # should cloud_5000 be "leader" and cloud_5100 be "repeater" etc?
   
-  # wish we had zkServerDataDir as a EnvUtil instead of system property!
+  export clusters_dir="${BATS_TEST_TMPDIR}/clusters"
   
-  mkdir -p cloud_5000
-  mkdir -p cloud_5100
-  mkdir -p cloud_5200
+  mkdir -p ${clusters_dir}/cluster_5000
+  mkdir -p ${clusters_dir}/cluster_5100
+  mkdir -p ${clusters_dir}/cluster_5200
   
   # Get our three seperate independent Solr nodes running.
-  bin/solr start -f -c -p 5000 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5000 -DzkServerDataDir=$(pwd)/cloud_5000/zoo_data -v -V 
-  bin/solr start -f -c -p 5100 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5100 -DzkServerDataDir=$(pwd)/cloud_5100/zoo_data -v -V 
-  bin/solr start -f -c -p 5200 -Dsolr.disable.allowUrls=true -s $(pwd)/cloud_5200 -DzkServerDataDir=$(pwd)/cloud_5200/zoo_data -v -V 
+  solr start -c -p 5000 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5000 -DzkServerDataDir=${clusters_dir}/cluster_5000/zoo_data -v -V 
+  solr start -c -p 5100 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5100 -DzkServerDataDir=${clusters_dir}/cluster_5100/zoo_data -v -V 
+  solr start -c -p 5200 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5200 -DzkServerDataDir=${clusters_dir}/cluster_5200/zoo_data -v -V 
   
   solr assert --started http://localhost:5000 --timeout 5000
   solr assert --started http://localhost:5100 --timeout 5000
@@ -54,11 +55,12 @@ teardown() {
   solr assert -cloud http://localhost:5200  
   
   # Wish I loaded configset seperately...
-  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5000
-  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5100
-  bin/solr create -c techproducts -d ./server/solr/configsets/sample_techproducts_configs -solrUrl http://localhost:5200
+  local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5000
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5100
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5200
   
-  # Verify no data
+  # Verify empty state of all the nodes
   run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":0'
   run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
@@ -67,7 +69,7 @@ teardown() {
   assert_output --partial '"numFound":0'  
   
   # Load XML formatted data into the leader
-  bin/solr post -type application/xml -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.xml
+  solr post -type application/xml -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.xml
   run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":32'
   
@@ -91,9 +93,7 @@ teardown() {
   run curl 'http://localhost:5000/solr/techproducts/replication?command=details'
   assert_output --partial '"replicationEnabled":"true"'
   
-  
-  # Setup the Repeater for replication
-  
+  # Setup the Repeater for replication  
   run curl -X POST -H 'Content-type:application/json' -d '{
     "add-requesthandler": {
       "name": "/replication",
@@ -107,30 +107,27 @@ teardown() {
   
   run curl 'http://localhost:5100/solr/techproducts/replication?command=details'
   assert_output --partial '"isPollingDisabled":"false"'
-
-  
-  # Wish we could block on fetchindex..   Does checking details help?
+ 
+ # How can we know when a replication has happened and then check?
   sleep 5
   run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":32' 
   
   # Testing adding new data by adding JSON formatted data into the leader
-  bin/solr post -type application/json -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.json
+  solr post -type application/json -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.json
   run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
   assert_output --partial '"numFound":36'
-  
   sleep 5
+  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  assert_output --partial '"numFound":36' 
   
   # Testing adding new data by adding CSV formatted data into the leader
-  bin/solr post -commit -url http://localhost:5000/solr/techproducts/update ./example/exampledocs/*.csv
+  solr post -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.csv
   run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
-  assert_output --partial '"numFound":46'
-  
-  sleep 5
-  
+  assert_output --partial '"numFound":46'  
+  sleep 5  
   run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
-  assert_output --partial '"numFound":46'
-  
+  assert_output --partial '"numFound":46'  
   
   run bash -c 'solr stop -all 2>&1'
   refute_output --partial 'forcefully killing'

From 10cec455a20bf0e60f3bbe5cca3bc3d04c8ed284 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 13:16:02 -0500
Subject: [PATCH 07/14] It ran end to end!

---
 solr/packaging/test/test_replication.bats | 76 ++++++++++++-----------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 56d6371448a2..52216b1e27d0 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -28,11 +28,12 @@ teardown() {
   solr stop -all >/dev/null 2>&1
 }
 
-@test "user managed index replication with a twist" {
-  export SOLR_SECURITY_MANAGER_ENABLED=false
+@test "user managed index replication with a twist" {  
   # This demonstrates traditional user managed cluster working as defined in
   # https://solr.apache.org/guide/solr/latest/deployment-guide/cluster-types.html 
   
+  export SOLR_SECURITY_MANAGER_ENABLED=false
+  
   # should cloud_5000 be "leader" and cloud_5100 be "repeater" etc?
   
   export clusters_dir="${BATS_TEST_TMPDIR}/clusters"
@@ -42,41 +43,41 @@ teardown() {
   mkdir -p ${clusters_dir}/cluster_5200
   
   # Get our three seperate independent Solr nodes running.
-  solr start -c -p 5000 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5000 -DzkServerDataDir=${clusters_dir}/cluster_5000/zoo_data -v -V 
-  solr start -c -p 5100 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5100 -DzkServerDataDir=${clusters_dir}/cluster_5100/zoo_data -v -V 
-  solr start -c -p 5200 -Dsolr.disable.allowUrls=true -s ${clusters_dir}/cluster_5200 -DzkServerDataDir=${clusters_dir}/cluster_5200/zoo_data -v -V 
+  solr start -c -p ${SOLR_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5000 -DzkServerDataDir="${clusters_dir}"/cluster_5000/zoo_data -v -V 
+  solr start -c -p ${SOLR2_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5100 -DzkServerDataDir="${clusters_dir}"/cluster_5100/zoo_data -v -V 
+  solr start -c -p ${SOLR3_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5200 -DzkServerDataDir="${clusters_dir}"/cluster_5200/zoo_data -v -V 
   
-  solr assert --started http://localhost:5000 --timeout 5000
-  solr assert --started http://localhost:5100 --timeout 5000
-  solr assert --started http://localhost:5200 --timeout 5000
+  solr assert --started http://localhost:${SOLR_PORT} --timeout 5000
+  solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000
+  solr assert --started http://localhost:${SOLR3_PORT} --timeout 5000
   
-  solr assert -cloud http://localhost:5000
-  solr assert -cloud http://localhost:5100
-  solr assert -cloud http://localhost:5200  
+  solr assert -cloud http://localhost:${SOLR_PORT}
+  solr assert -cloud http://localhost:${SOLR2_PORT}
+  solr assert -cloud http://localhost:${SOLR3_PORT}  
   
   # Wish I loaded configset seperately...
   local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5000
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5100
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:5200
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR_PORT}
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR2_PORT}
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR3_PORT}
   
   # Verify empty state of all the nodes
-  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'
-  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'
-  run curl 'http://localhost:5200/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
   
   # Load XML formatted data into the leader
-  solr post -type application/xml -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.xml
-  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  solr post -type application/xml -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.xml
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32'
   
   # Confirm no replication
-  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
-  run curl 'http://localhost:5200/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
   
   # Setup the Leader for replication
@@ -84,13 +85,13 @@ teardown() {
     "add-requesthandler": {
       "name": "/replication",
       "class": "solr.ReplicationHandler",
-      "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
+      "leader":{ "replicateAfter": "commit", "replicateAfter": "optimize", "backupAfter":"commit", "confFiles":""},
       "maxNumberOfBackups":2            
     }
-  }' "http://localhost:5000/solr/techproducts/config"
+  }' "http://localhost:${SOLR_PORT}/solr/techproducts/config"
   assert_output --partial '"status":0'
   
-  run curl 'http://localhost:5000/solr/techproducts/replication?command=details'
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"replicationEnabled":"true"'
   
   # Setup the Repeater for replication  
@@ -98,35 +99,38 @@ teardown() {
     "add-requesthandler": {
       "name": "/replication",
       "class": "solr.ReplicationHandler",
-      "follower":{ "leaderUrl": "http://localhost:5000/solr/techproducts/replication", "pollInterval":"00:00:02"},
+      "follower":{ "leaderUrl": "http://localhost:'"${SOLR_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"},
       "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
       "maxNumberOfBackups":2            
     }
-  }' "http://localhost:5100/solr/techproducts/config"
+  }' "http://localhost:${SOLR2_PORT}/solr/techproducts/config"
   assert_output --partial '"status":0'
   
-  run curl 'http://localhost:5100/solr/techproducts/replication?command=details'
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"isPollingDisabled":"false"'
  
- # How can we know when a replication has happened and then check?
-  sleep 5
-  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  # How can we know when a replication has happened and then check?
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/update?optimize=true"
+  assert_output --partial '"status":0'
+  sleep 10
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/replication?command=details"
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32' 
   
   # Testing adding new data by adding JSON formatted data into the leader
-  solr post -type application/json -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.json
-  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  solr post -type application/json -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.json
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":36'
   sleep 5
-  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":36' 
   
   # Testing adding new data by adding CSV formatted data into the leader
-  solr post -commit -url http://localhost:5000/solr/techproducts/update ${SOLR_TIP}/example/exampledocs/*.csv
-  run curl 'http://localhost:5000/solr/techproducts/select?q=*:*&rows=0'
+  solr post -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.csv
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   sleep 5  
-  run curl 'http://localhost:5100/solr/techproducts/select?q=*:*&rows=0'
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   
   run bash -c 'solr stop -all 2>&1'

From dcd1599a5256d55437c0c52d64c3428ae6468352 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 13:59:08 -0500
Subject: [PATCH 08/14] Now we have a leader/replicator/follower setup!

---
 solr/packaging/test/test_replication.bats | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 52216b1e27d0..7fd18ebef3e0 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -85,7 +85,7 @@ teardown() {
     "add-requesthandler": {
       "name": "/replication",
       "class": "solr.ReplicationHandler",
-      "leader":{ "replicateAfter": "commit", "replicateAfter": "optimize", "backupAfter":"commit", "confFiles":""},
+      "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
       "maxNumberOfBackups":2            
     }
   }' "http://localhost:${SOLR_PORT}/solr/techproducts/config"
@@ -108,12 +108,24 @@ teardown() {
   
   run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"isPollingDisabled":"false"'
+  
+  # Setup the Follower for replication  
+  run curl -X POST -H 'Content-type:application/json' -d '{
+    "add-requesthandler": {
+      "name": "/replication",
+      "class": "solr.ReplicationHandler",
+      "follower":{ "leaderUrl": "http://localhost:'"${SOLR2_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"}         
+    }
+  }' "http://localhost:${SOLR3_PORT}/solr/techproducts/config"
+  assert_output --partial '"status":0'
+  
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/replication?command=details"
+  assert_output --partial '"isPollingDisabled":"false"'  
  
   # How can we know when a replication has happened and then check?
   run curl "http://localhost:${SOLR_PORT}/solr/techproducts/update?optimize=true"
   assert_output --partial '"status":0'
   sleep 10
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/replication?command=details"
   run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32' 
   
@@ -130,9 +142,16 @@ teardown() {
   run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   sleep 5  
+  echo "Waiting to see Solr2 on ${SOLR2_PORT} update"
+  sleep 20
   run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   
+  # Now lets go check our Follower and make sure it's picks up all the changes too!
+  sleep 5  
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  assert_output --partial '"numFound":46' 
+  
   run bash -c 'solr stop -all 2>&1'
   refute_output --partial 'forcefully killing'
 }

From 759792eabde1ea8acf017184cd8934377b2e381e Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 14:21:40 -0500
Subject: [PATCH 09/14] Demonstrate stopping a repeater, bringing it back up
 and it preserves it's configuration, and then it starts repeating data.

---
 solr/packaging/test/test_replication.bats | 40 +++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 7fd18ebef3e0..6e8b87f89cc0 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -31,6 +31,17 @@ teardown() {
 @test "user managed index replication with a twist" {  
   # This demonstrates traditional user managed cluster working as defined in
   # https://solr.apache.org/guide/solr/latest/deployment-guide/cluster-types.html 
+  #
+  # We demonstrate starting up three independent Solr nodes in the Leader/Repeater/Follower pattern.
+  # Then we create three seperate 'techproducts' collections, uploading the same configset three seperate times
+  # to demonstrate that there is no interconnection or shard config between them.
+  # We then index some XML data on the Leader, and then check that it flows through the Repeater to the Follower.
+  # This is repeated for some more documents.
+  # Lastly, we shutdown the Repeater and demonstrate that the Follower still has all of it's documents available for querying.
+  # We delete the data on the Leader, and then subsequantly bring back up the Repeater.
+  # The Repeater perseves all fo the configuration that was done during the setup process after restarting, and immediatley copies
+  # over the now empty 'techproducts' index and we then see the Follower picks up that empty collection as well.
+  
   
   export SOLR_SECURITY_MANAGER_ENABLED=false
   
@@ -152,6 +163,35 @@ teardown() {
   run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46' 
   
+  # Now lets stop our replicator
+  solr stop -p ${SOLR2_PORT}
+  
+  solr assert --not-started http://localhost:${SOLR2_PORT} --timeout 5000
+  
+  # Delete data on the leader.
+  solr post -url http://localhost:${SOLR_PORT}/solr/techproducts/update -mode args -out -commit "{'delete': {'query': '*:*'}}"
+  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  assert_output --partial '"numFound":0' 
+
+  # check our follower is still up and responding
+  sleep 5  
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  assert_output --partial '"numFound":46' 
+  
+  # Bring back our Repeater
+  solr start -c -p ${SOLR2_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5100 -DzkServerDataDir="${clusters_dir}"/cluster_5100/zoo_data -v -V 
+  solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000
+  
+  # check our Repeater is picking up the deleted documents from the Leader
+  sleep 5  
+  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  assert_output --partial '"numFound":0' 
+  
+  # And now check our follower has no documents as well.
+  sleep 5  
+  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  assert_output --partial '"numFound":0' 
+  
   run bash -c 'solr stop -all 2>&1'
   refute_output --partial 'forcefully killing'
 }

From 2f3aae4379f360e38b0f634dd24bf6707f3a27e6 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 14:28:17 -0500
Subject: [PATCH 10/14] Back out changes made to support standalone solr
 because what I really want is solr cloud!

---
 solr/bin/solr                                      |  1 -
 .../java/org/apache/solr/cli/RunExampleTool.java   | 14 +-------------
 .../org/apache/solr/cli/TestSolrCLIRunExample.java | 14 ++++----------
 3 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/solr/bin/solr b/solr/bin/solr
index bf580867d59e..0e2f93ae348e 100755
--- a/solr/bin/solr
+++ b/solr/bin/solr
@@ -1283,7 +1283,6 @@ if [ $# -gt 0 ]; then
         ;;
         -t|-data.home)
             SOLR_DATA_HOME="$2"
-            PASS_TO_RUN_EXAMPLE+=("-t" "$SOLR_DATA_HOME")
             shift 2
         ;;
         -e|-example)
diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index 080b8e5e53ae..f77ae4cb202f 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -118,13 +118,6 @@ public List<Option> getOptions() {
             .desc("Path to the Solr server directory.")
             .longOpt("serverDir")
             .build(),
-        Option.builder("t")
-            .argName("DIR")
-            .hasArg()
-            .required(false)
-            .desc("Path to the Solr data home directory.")
-            .longOpt("dataHomeDir")
-            .build(),
         Option.builder("force")
             .argName("FORCE")
             .desc("Force option in case Solr is run as root.")
@@ -615,9 +608,6 @@ protected Map<String, Object> startSolr(
     String forceArg = cli.hasOption("force") ? " -force" : "";
     String verboseArg = verbose ? "-V" : "";
 
-    String dataDirArg = cli.hasOption("dataHomeDir") ? "-t" : "";
-    String dataDirValue = cli.hasOption("dataHomeDir") ? cli.getOptionValue("dataHomeDir") : "";
-
     String addlOpts = cli.getOptionValue('a');
     String addlOptsArg = (addlOpts != null) ? " -a \"" + addlOpts + "\"" : "";
 
@@ -637,7 +627,7 @@ protected Map<String, Object> startSolr(
     String startCmd =
         String.format(
             Locale.ROOT,
-            "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s %s %s %s",
+            "\"%s\" start %s -p %d -s \"%s\" %s %s %s %s %s %s %s",
             callScript,
             cloudModeArg,
             port,
@@ -647,8 +637,6 @@ protected Map<String, Object> startSolr(
             memArg,
             forceArg,
             verboseArg,
-            dataDirArg,
-            dataDirValue,
             extraArgs,
             addlOptsArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
diff --git a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
index c6c89ce63508..ac2ac704694f 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
@@ -353,16 +353,10 @@ protected void testExample(String exampleName) throws Exception {
 
       String[] toolArgs =
           new String[] {
-            "-e",
-            exampleName,
-            "-serverDir",
-            solrServerDir.getAbsolutePath(),
-            "-exampleDir",
-            solrExampleDir.getAbsolutePath(),
-            "-p",
-            String.valueOf(bindPort),
-            "-t",
-            "dude"
+            "-e", exampleName,
+            "-serverDir", solrServerDir.getAbsolutePath(),
+            "-exampleDir", solrExampleDir.getAbsolutePath(),
+            "-p", String.valueOf(bindPort)
           };
 
       // capture tool output to stdout

From fb15f80d18bac0402f9e36832f7619305c18f841 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Sun, 11 Feb 2024 14:29:12 -0500
Subject: [PATCH 11/14] restore nightly

---
 .../src/test/org/apache/solr/cli/TestSolrCLIRunExample.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
index ac2ac704694f..475770824284 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java
@@ -316,7 +316,7 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  // @LuceneTestCase.Nightly
+  @LuceneTestCase.Nightly
   public void testTechproductsExample() throws Exception {
     testExample("techproducts");
   }

From 552c454f3563d8fb25b258afc39e7d42036bdb53 Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Mon, 19 Feb 2024 16:53:15 -0500
Subject: [PATCH 12/14] rename port to try and tell a clearer story

---
 solr/packaging/test/test_replication.bats | 126 +++++++++++-----------
 1 file changed, 63 insertions(+), 63 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 6e8b87f89cc0..931aeeda85f2 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -39,59 +39,62 @@ teardown() {
   # This is repeated for some more documents.
   # Lastly, we shutdown the Repeater and demonstrate that the Follower still has all of it's documents available for querying.
   # We delete the data on the Leader, and then subsequantly bring back up the Repeater.
-  # The Repeater perseves all fo the configuration that was done during the setup process after restarting, and immediatley copies
+  # The Repeater preserves all fo the configuration that was done during the setup process after restarting, and immediatley copies
   # over the now empty 'techproducts' index and we then see the Follower picks up that empty collection as well.
   
-  
   export SOLR_SECURITY_MANAGER_ENABLED=false
   
-  # should cloud_5000 be "leader" and cloud_5100 be "repeater" etc?
+  LEADER_PORT=$SOLR_PORT
+  REPEATER_PORT=$SOLR2_PORT
+  FOLLOWER_PORT=$SOLR3_PORT
   
   export clusters_dir="${BATS_TEST_TMPDIR}/clusters"
   
-  mkdir -p ${clusters_dir}/cluster_5000
-  mkdir -p ${clusters_dir}/cluster_5100
-  mkdir -p ${clusters_dir}/cluster_5200
+  mkdir -p ${clusters_dir}/leader
+  mkdir -p ${clusters_dir}/repeater
+  mkdir -p ${clusters_dir}/follower
+  
+
   
   # Get our three seperate independent Solr nodes running.
-  solr start -c -p ${SOLR_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5000 -DzkServerDataDir="${clusters_dir}"/cluster_5000/zoo_data -v -V 
-  solr start -c -p ${SOLR2_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5100 -DzkServerDataDir="${clusters_dir}"/cluster_5100/zoo_data -v -V 
-  solr start -c -p ${SOLR3_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5200 -DzkServerDataDir="${clusters_dir}"/cluster_5200/zoo_data -v -V 
+  solr start -c -p ${LEADER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/leader -DzkServerDataDir="${clusters_dir}"/leader/zoo_data -v -V 
+  solr start -c -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data -v -V 
+  solr start -c -p ${FOLLOWER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/follower -DzkServerDataDir="${clusters_dir}"/follower/zoo_data -v -V 
   
-  solr assert --started http://localhost:${SOLR_PORT} --timeout 5000
-  solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000
-  solr assert --started http://localhost:${SOLR3_PORT} --timeout 5000
+  solr assert --started http://localhost:${LEADER_PORT} --timeout 5000
+  solr assert --started http://localhost:${REPEATER_PORT} --timeout 5000
+  solr assert --started http://localhost:${FOLLOWER_PORT} --timeout 5000
   
-  solr assert -cloud http://localhost:${SOLR_PORT}
-  solr assert -cloud http://localhost:${SOLR2_PORT}
-  solr assert -cloud http://localhost:${SOLR3_PORT}  
+  solr assert -cloud http://localhost:${LEADER_PORT}
+  solr assert -cloud http://localhost:${REPEATER_PORT}
+  solr assert -cloud http://localhost:${FOLLOWER_PORT}  
   
   # Wish I loaded configset seperately...
   local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR_PORT}
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR2_PORT}
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${SOLR3_PORT}
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${LEADER_PORT}
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${REPEATER_PORT}
+  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${FOLLOWER_PORT}
   
-  # Verify empty state of all the nodes
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  # Verify empty state of all the clusters
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
   
   # Load XML formatted data into the leader
-  solr post -type application/xml -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.xml
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  solr post -type application/xml -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.xml
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32'
   
-  # Confirm no replication
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  # Confirm no replication is taking place
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0'  
   
-  # Setup the Leader for replication
+  # Setup the leader for replication
   run curl -X POST -H 'Content-type:application/json' -d '{
     "add-requesthandler": {
       "name": "/replication",
@@ -99,10 +102,10 @@ teardown() {
       "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
       "maxNumberOfBackups":2            
     }
-  }' "http://localhost:${SOLR_PORT}/solr/techproducts/config"
+  }' "http://localhost:${LEADER_PORT}/solr/techproducts/config"
   assert_output --partial '"status":0'
   
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/replication?command=details"
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"replicationEnabled":"true"'
   
   # Setup the Repeater for replication  
@@ -110,14 +113,14 @@ teardown() {
     "add-requesthandler": {
       "name": "/replication",
       "class": "solr.ReplicationHandler",
-      "follower":{ "leaderUrl": "http://localhost:'"${SOLR_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"},
+      "follower":{ "leaderUrl": "http://localhost:'"${LEADER_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"},
       "leader":{ "replicateAfter": "commit", "backupAfter":"commit", "confFiles":""},
       "maxNumberOfBackups":2            
     }
-  }' "http://localhost:${SOLR2_PORT}/solr/techproducts/config"
+  }' "http://localhost:${REPEATER_PORT}/solr/techproducts/config"
   assert_output --partial '"status":0'
   
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/replication?command=details"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"isPollingDisabled":"false"'
   
   # Setup the Follower for replication  
@@ -125,71 +128,68 @@ teardown() {
     "add-requesthandler": {
       "name": "/replication",
       "class": "solr.ReplicationHandler",
-      "follower":{ "leaderUrl": "http://localhost:'"${SOLR2_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"}         
+      "follower":{ "leaderUrl": "http://localhost:'"${REPEATER_PORT}"'/solr/techproducts/replication", "pollInterval":"00:00:02"}         
     }
-  }' "http://localhost:${SOLR3_PORT}/solr/techproducts/config"
+  }' "http://localhost:${FOLLOWER_PORT}/solr/techproducts/config"
   assert_output --partial '"status":0'
   
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/replication?command=details"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/replication?command=details"
   assert_output --partial '"isPollingDisabled":"false"'  
  
   # How can we know when a replication has happened and then check?
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/update?optimize=true"
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/update?optimize=true"
   assert_output --partial '"status":0'
   sleep 10
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32' 
   
   # Testing adding new data by adding JSON formatted data into the leader
-  solr post -type application/json -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.json
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  solr post -type application/json -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.json
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":36'
   sleep 5
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":36' 
   
   # Testing adding new data by adding CSV formatted data into the leader
-  solr post -commit -url http://localhost:${SOLR_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.csv
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  solr post -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.csv
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
-  sleep 5  
-  echo "Waiting to see Solr2 on ${SOLR2_PORT} update"
   sleep 20
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   
-  # Now lets go check our Follower and make sure it's picks up all the changes too!
+  # Now lets go check our follower and make sure it picks up all the changes too!
   sleep 5  
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46' 
   
-  # Now lets stop our replicator
-  solr stop -p ${SOLR2_PORT}
-  
-  solr assert --not-started http://localhost:${SOLR2_PORT} --timeout 5000
+  # Now lets stop our repeater
+  solr stop -p ${REPEATER_PORT}
+  solr assert --not-started http://localhost:${REPEATER_PORT} --timeout 5000
   
-  # Delete data on the leader.
-  solr post -url http://localhost:${SOLR_PORT}/solr/techproducts/update -mode args -out -commit "{'delete': {'query': '*:*'}}"
-  run curl "http://localhost:${SOLR_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  # Delete data on the leader
+  solr post -url http://localhost:${LEADER_PORT}/solr/techproducts/update -mode args -out -commit "{'delete': {'query': '*:*'}}"
+  run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0' 
 
   # check our follower is still up and responding
   sleep 5  
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46' 
   
-  # Bring back our Repeater
-  solr start -c -p ${SOLR2_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/cluster_5100 -DzkServerDataDir="${clusters_dir}"/cluster_5100/zoo_data -v -V 
-  solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000
+  # Bring back our repeater
+  solr start -c -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data -v -V 
+  solr assert --started http://localhost:${REPEATER_PORT} --timeout 5000
   
-  # check our Repeater is picking up the deleted documents from the Leader
+  # check our repeater is picking up the now deleted documents from the leader
   sleep 5  
-  run curl "http://localhost:${SOLR2_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${REPEATER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0' 
   
-  # And now check our follower has no documents as well.
+  # And check our follower has no documents as well.
   sleep 5  
-  run curl "http://localhost:${SOLR3_PORT}/solr/techproducts/select?q=*:*&rows=0"
+  run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0' 
   
   run bash -c 'solr stop -all 2>&1'

From 57c3e13d918e3a1d48c9cb19259a06557e5eb7cf Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Fri, 1 Nov 2024 12:50:58 -0400
Subject: [PATCH 13/14] update cli options

---
 solr/packaging/test/test_replication.bats | 30 +++++++++++------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_replication.bats
index 931aeeda85f2..8f1647d40208 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_replication.bats
@@ -57,23 +57,23 @@ teardown() {
 
   
   # Get our three seperate independent Solr nodes running.
-  solr start -c -p ${LEADER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/leader -DzkServerDataDir="${clusters_dir}"/leader/zoo_data -v -V 
-  solr start -c -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data -v -V 
-  solr start -c -p ${FOLLOWER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/follower -DzkServerDataDir="${clusters_dir}"/follower/zoo_data -v -V 
+  solr start -p ${LEADER_PORT} -Dsolr.disable.allowUrls=true --solr-home "${clusters_dir}"/leader -DzkServerDataDir="${clusters_dir}"/leader/zoo_data --verbose
+  solr start -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true --solr-home "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data --verbose
+  solr start -p ${FOLLOWER_PORT} -Dsolr.disable.allowUrls=true --solr-home "${clusters_dir}"/follower -DzkServerDataDir="${clusters_dir}"/follower/zoo_data --verbose 
   
   solr assert --started http://localhost:${LEADER_PORT} --timeout 5000
   solr assert --started http://localhost:${REPEATER_PORT} --timeout 5000
   solr assert --started http://localhost:${FOLLOWER_PORT} --timeout 5000
   
-  solr assert -cloud http://localhost:${LEADER_PORT}
-  solr assert -cloud http://localhost:${REPEATER_PORT}
-  solr assert -cloud http://localhost:${FOLLOWER_PORT}  
+  solr assert --cloud http://localhost:${LEADER_PORT}
+  solr assert --cloud http://localhost:${REPEATER_PORT}
+  solr assert --cloud http://localhost:${FOLLOWER_PORT}  
   
   # Wish I loaded configset seperately...
   local source_configset_dir="${SOLR_TIP}/server/solr/configsets/sample_techproducts_configs"
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${LEADER_PORT}
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${REPEATER_PORT}
-  solr create -c techproducts -d "${source_configset_dir}" -solrUrl http://localhost:${FOLLOWER_PORT}
+  solr create -c techproducts --conf-dir "${source_configset_dir}" --solr-url http://localhost:${LEADER_PORT}
+  solr create -c techproducts --conf-dir "${source_configset_dir}" --solr-url http://localhost:${REPEATER_PORT}
+  solr create -c techproducts --conf-dir "${source_configset_dir}" --solr-url http://localhost:${FOLLOWER_PORT}
   
   # Verify empty state of all the clusters
   run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
@@ -84,7 +84,7 @@ teardown() {
   assert_output --partial '"numFound":0'  
   
   # Load XML formatted data into the leader
-  solr post -type application/xml -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.xml
+  solr post --type application/xml --solr-url http://localhost:${LEADER_PORT} -c techproducts "${SOLR_TIP}"/example/exampledocs/*.xml
   run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":32'
   
@@ -144,7 +144,7 @@ teardown() {
   assert_output --partial '"numFound":32' 
   
   # Testing adding new data by adding JSON formatted data into the leader
-  solr post -type application/json -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.json
+  solr post --type application/json --solr-url http://localhost:${LEADER_PORT} -c techproducts "${SOLR_TIP}"/example/exampledocs/*.json
   run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":36'
   sleep 5
@@ -152,7 +152,7 @@ teardown() {
   assert_output --partial '"numFound":36' 
   
   # Testing adding new data by adding CSV formatted data into the leader
-  solr post -commit -url http://localhost:${LEADER_PORT}/solr/techproducts/update "${SOLR_TIP}"/example/exampledocs/*.csv
+  solr post --solr-url http://localhost:${LEADER_PORT} -c techproducts "${SOLR_TIP}"/example/exampledocs/*.csv
   run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":46'  
   sleep 20
@@ -169,7 +169,7 @@ teardown() {
   solr assert --not-started http://localhost:${REPEATER_PORT} --timeout 5000
   
   # Delete data on the leader
-  solr post -url http://localhost:${LEADER_PORT}/solr/techproducts/update -mode args -out -commit "{'delete': {'query': '*:*'}}"
+  solr post --solr-url http://localhost:${LEADER_PORT} -c techproducts  --mode args --verbose "{'delete': {'query': '*:*'}}"
   run curl "http://localhost:${LEADER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0' 
 
@@ -179,7 +179,7 @@ teardown() {
   assert_output --partial '"numFound":46' 
   
   # Bring back our repeater
-  solr start -c -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true -s "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data -v -V 
+  solr start -p ${REPEATER_PORT} -Dsolr.disable.allowUrls=true --solr-home "${clusters_dir}"/repeater -DzkServerDataDir="${clusters_dir}"/repeater/zoo_data --verbose
   solr assert --started http://localhost:${REPEATER_PORT} --timeout 5000
   
   # check our repeater is picking up the now deleted documents from the leader
@@ -192,6 +192,4 @@ teardown() {
   run curl "http://localhost:${FOLLOWER_PORT}/solr/techproducts/select?q=*:*&rows=0"
   assert_output --partial '"numFound":0' 
   
-  run bash -c 'solr stop -all 2>&1'
-  refute_output --partial 'forcefully killing'
 }

From ddcfcdff6746b0df8b4a6a13d527f75201f74c3a Mon Sep 17 00:00:00 2001
From: Eric Pugh <epugh@opensourceconnections.com>
Date: Fri, 1 Nov 2024 12:54:25 -0400
Subject: [PATCH 14/14] Introduce idea of a scenario.

In PR 2783 we talk about various approaches to deploying solr from small to large.  It would be good to actually test those deployment scenarios.  This tries the first one out.
---
 ...ation.bats => test_scenario_loosely_coupled_clusters.bats} | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
 rename solr/packaging/test/{test_replication.bats => test_scenario_loosely_coupled_clusters.bats} (97%)

diff --git a/solr/packaging/test/test_replication.bats b/solr/packaging/test/test_scenario_loosely_coupled_clusters.bats
similarity index 97%
rename from solr/packaging/test/test_replication.bats
rename to solr/packaging/test/test_scenario_loosely_coupled_clusters.bats
index 8f1647d40208..37b55a4477d0 100644
--- a/solr/packaging/test/test_replication.bats
+++ b/solr/packaging/test/test_scenario_loosely_coupled_clusters.bats
@@ -32,7 +32,9 @@ teardown() {
   # This demonstrates traditional user managed cluster working as defined in
   # https://solr.apache.org/guide/solr/latest/deployment-guide/cluster-types.html 
   #
-  # We demonstrate starting up three independent Solr nodes in the Leader/Repeater/Follower pattern.
+  # See "Loosely coupled Solr Nodes" in https://github.com/apache/solr/pull/2783/files#diff-b58818a370dac65f7abb0064599f8813a56841b4a40f960ea2b81e398b820f43
+  #
+  # We demonstrate starting up three independent SolrCloud nodes in the Leader/Repeater/Follower pattern.
   # Then we create three seperate 'techproducts' collections, uploading the same configset three seperate times
   # to demonstrate that there is no interconnection or shard config between them.
   # We then index some XML data on the Leader, and then check that it flows through the Repeater to the Follower.