From 1e63065032ae049d72ada434c6a87cbb44c6528f Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sat, 18 Jan 2025 13:16:04 -0500 Subject: [PATCH 01/64] add logchange plugin --- build.gradle | 1 + gradle/libs.versions.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index 2ff703ce8f58..055759546ab0 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,7 @@ plugins { alias(libs.plugins.diffplug.spotless) apply false alias(libs.plugins.nodegradle.node) apply false alias(libs.plugins.openapi.generator) apply false + alias(libs.plugins.logchange) apply false } // Declare default Java versions for the entire project and for SolrJ separately diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 685fe0122165..f07c822c60b0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -184,6 +184,7 @@ threeten-bp = "1.6.8" undercouch-download = "5.5.0" xerces = "2.12.2" xerial-snappy = "1.1.10.5" +logchange = "1.16.4" [plugins] benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } @@ -199,6 +200,7 @@ owasp-dependencycheck = { id = "org.owasp.dependencycheck", version.ref = "owasp swagger3-core = { id = "io.swagger.core.v3.swagger-gradle-plugin", version.ref = "swagger3" } thetaphi-forbiddenapis = { id = "de.thetaphi.forbiddenapis", version.ref = "thetaphi-forbiddenapis" } undercouch-download = { id = "de.undercouch.download", version.ref = "undercouch-download" } +logchange = { id = 'dev.logchange', version.ref = "logchange" } [libraries] adobe-testing-s3mock-junit4 = { module = "com.adobe.testing:s3mock-junit4", version.ref = "adobe-testing-s3mock" } From b38a7b29fe22d10ab310f6d3f18882b9b85ad419 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 20 Jan 2025 12:11:03 -0500 Subject: [PATCH 02/64] Fix --- build.gradle | 13 ++++++++++++- changelog/unreleased/.keep | 0 gradle/libs.versions.toml | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/.keep diff --git a/build.gradle b/build.gradle index 055759546ab0..059caf223634 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ plugins { alias(libs.plugins.diffplug.spotless) apply false alias(libs.plugins.nodegradle.node) apply false alias(libs.plugins.openapi.generator) apply false - alias(libs.plugins.logchange) apply false + alias(libs.plugins.logchange) } // Declare default Java versions for the entire project and for SolrJ separately @@ -202,3 +202,14 @@ apply from: file('gradle/solr/packaging.gradle') apply from: file('gradle/solr/solr-forbidden-apis.gradle') apply from: file('gradle/node.gradle') + +logchange { + // taskPrefix = "logchange" + rootPath = "." + inputDir = "changelog" + unreleasedVersionDir = "unreleased" + outputFile = "CHANGELOG.md" + configFile = "logchange-config.yml" + generateChangesXml = false + xmlOutputFile = "changes.xml" +} \ No newline at end of file diff --git a/changelog/unreleased/.keep b/changelog/unreleased/.keep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f07c822c60b0..a979f19c8e49 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -184,7 +184,7 @@ threeten-bp = "1.6.8" undercouch-download = "5.5.0" xerces = "2.12.2" xerial-snappy = "1.1.10.5" -logchange = "1.16.4" +logchange = "1.16.5" [plugins] benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } From 3cc1497ae9dd22a9937eba3cf919ae9ff502b87f Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 20 Jan 2025 13:59:20 -0500 Subject: [PATCH 03/64] Task generateChangeYaml And move logchange config --- build.gradle | 11 +-------- gradle/changelog.gradle | 50 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 gradle/changelog.gradle diff --git a/build.gradle b/build.gradle index 059caf223634..11c8f1bc7daa 100644 --- a/build.gradle +++ b/build.gradle @@ -203,13 +203,4 @@ apply from: file('gradle/solr/solr-forbidden-apis.gradle') apply from: file('gradle/node.gradle') -logchange { - // taskPrefix = "logchange" - rootPath = "." - inputDir = "changelog" - unreleasedVersionDir = "unreleased" - outputFile = "CHANGELOG.md" - configFile = "logchange-config.yml" - generateChangesXml = false - xmlOutputFile = "changes.xml" -} \ No newline at end of file +apply from: file('gradle/changelog.gradle') diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle new file mode 100644 index 000000000000..1b482741f169 --- /dev/null +++ b/gradle/changelog.gradle @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +logchange { + taskPrefix = "logchange" + rootPath = "." + inputDir = "changelog" + unreleasedVersionDir = "unreleased" + outputFile = "CHANGELOG.md" + configFile = "logchange-config.yml" + generateChangesXml = false + xmlOutputFile = "changes.xml" +} + +task generateChangeYaml { + description = 'Generates a change/log description file (YAML)' + doLast { + def gitUserName = 'git config user.name'.execute().text.trim() + def gitBranch = 'git rev-parse --abbrev-ref HEAD'.execute().text.trim() + def jiraMatcher = gitBranch =~ /SOLR-\d+/ + def jiraRef = jiraMatcher ? jiraMatcher[0] : "SOLR-XXXX" + def fileName = "changelog/unreleased/${gitBranch}.yml" + def file = new File(fileName) + file.parentFile.mkdirs() + file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format +type: other # added/changed/deprecated/removed/fixed/security/dependency_update/other +links: + - name: ${jiraRef} +title: TITLE HERE +authors: + - name: ${gitUserName} +""" + + println "Generated file: ${fileName} -- open it" + } +} \ No newline at end of file From 5ba088f4029e80233041c8da51adcb649fdeb0ec Mon Sep 17 00:00:00 2001 From: David Smiley Date: Mon, 20 Jan 2025 21:53:25 -0500 Subject: [PATCH 04/64] sorted deps --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a979f19c8e49..476b10c5d072 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -146,6 +146,7 @@ langchain4j = "0.35.0" link-checker = "1.4.2" littlerobots-versioncatalogupdate = "0.8.4" lmax-disruptor = "3.4.4" +logchange = "1.16.5" ltgt-errorprone = "3.1.0" mockito = "5.12.0" morethan-jmhreport = "0.9.0" @@ -184,7 +185,6 @@ threeten-bp = "1.6.8" undercouch-download = "5.5.0" xerces = "2.12.2" xerial-snappy = "1.1.10.5" -logchange = "1.16.5" [plugins] benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } @@ -192,6 +192,7 @@ carrotsearch-dependencychecks = { id = "com.carrotsearch.gradle.dependencychecks cutterslade-analyze = { id = "ca.cutterslade.analyze", version.ref = "cutterslade-analyze" } diffplug-spotless = { id = "com.diffplug.spotless", version.ref = "diffplug-spotless" } littlerobots-versioncatalogupdate = { id = "nl.littlerobots.version-catalog-update", version.ref = "littlerobots-versioncatalogupdate" } +logchange = { id = "dev.logchange", version.ref = "logchange" } ltgt-errorprone = { id = "net.ltgt.errorprone", version.ref = "ltgt-errorprone" } morethan-jmhreport = { id = "io.morethan.jmhreport", version.ref = "morethan-jmhreport" } nodegradle-node = { id = "com.github.node-gradle.node", version.ref = "nodegradle-node" } @@ -200,7 +201,6 @@ owasp-dependencycheck = { id = "org.owasp.dependencycheck", version.ref = "owasp swagger3-core = { id = "io.swagger.core.v3.swagger-gradle-plugin", version.ref = "swagger3" } thetaphi-forbiddenapis = { id = "de.thetaphi.forbiddenapis", version.ref = "thetaphi-forbiddenapis" } undercouch-download = { id = "de.undercouch.download", version.ref = "undercouch-download" } -logchange = { id = 'dev.logchange', version.ref = "logchange" } [libraries] adobe-testing-s3mock-junit4 = { module = "com.adobe.testing:s3mock-junit4", version.ref = "adobe-testing-s3mock" } From 0421dc90e10613e5ff489ccf2a29315a645e6218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 01:31:02 +0200 Subject: [PATCH 05/64] Bump version --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 476b10c5d072..a49e4f5623ca 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -146,7 +146,7 @@ langchain4j = "0.35.0" link-checker = "1.4.2" littlerobots-versioncatalogupdate = "0.8.4" lmax-disruptor = "3.4.4" -logchange = "1.16.5" +logchange = "1.19.4" ltgt-errorprone = "3.1.0" mockito = "5.12.0" morethan-jmhreport = "0.9.0" From 04251c2c30af44cbec50911be314360d457a7a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 01:31:17 +0200 Subject: [PATCH 06/64] Add logchange-config.yml --- changelog/logchange-config.yml | 90 ++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 changelog/logchange-config.yml diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml new file mode 100644 index 000000000000..762152b99eb9 --- /dev/null +++ b/changelog/logchange-config.yml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# More info about configuration you can find https://github.com/logchange/logchange#configuration +changelog: + heading: Some information that will be display in the top of CHANGELOG.md + unreleasedVersionDir: unreleased + # You can define custom entry types, which overriders default ones + # [added/changed/deprecated/removed/fixed/security/dependency_update/other] + # Remember to adjust labels below, to match new entry types f.e. add: New features + # entryTypes: # you can define custom entry types + # - key: add + # order: 1 + # - key: fix + # order: 2 + # - key: change + # order: 3 + # - key: other + # order: 4 + labels: + unreleased: unreleased + important_notes: Important notes + types: + entryTypesLabels: + security: Security + other: Other + removed: Removed + added: Added + deprecated: Deprecated + fixed: Fixed + dependency_update: Dependency updates + changed: Changed + number_of_changes: + singular: change + plural: changes + security: Security + other: Other + removed: Removed + added: Added + deprecated: Deprecated + fixed: Fixed + dependency_update: Dependency updates + changed: Changed + configuration: + heading: Configuration changes + type: Type + actions: + add: Added + update: Updated + delete: Deleted + with_default_value: with default value + description: Description + templates: + entry: "${prefix}${title} ${merge_requests} ${issues} ${links} ${authors}" + author: "([${name}](${url}) @${nick})" + # see examples of templates at examples/templates + # if you are missing some function, which will simplify your template (f.e getNumberOfEntries()) + # feel free to create issue or pull request with change +# version_summary_templates: + # Relative path to the changelog/.templates directory. + # Following definition will require from you existence of template at + # changelog/.templates/my-version-summary.html + # It will create my-version-summary.html in every version directory + # changelog/vX.X.X/my-version-summary.html + # HOW TO CREATE VERSION SUMMARY TEMPLATE? + # Main object is version and its type of https://github.com/logchange/logchange/blob/main/logchange-core/src/main/java/dev/logchange/core/domain/changelog/model/version/ChangelogVersion.java +# - path: my-version-summary.html +# changelog_templates: + # Relative path to the changelog/.templates directory. + # Following definition will require from you existence of template at + # changelog/.templates/my-changelog.html + # It will create my-changelog.html in every version directory + # changelog/vX.X.X/my-version-summary.html + # HOW TO CREATE CHANGELOG TEMPLATE? + # Main object is changelog and its type of https://github.com/logchange/logchange/blob/main/logchange-core/src/main/java/dev/logchange/core/domain/changelog/model/Changelog.java +# - path: my-changelog.html From e6a613263977fb2ba1387feb5405c8deb4876481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 02:42:47 +0200 Subject: [PATCH 07/64] Dev-doc and gradle.properties template --- dev-docs/changelog.adoc | 116 ++++++++++++++++++++++++++++++ gradle/template.gradle.properties | 5 ++ 2 files changed, 121 insertions(+) create mode 100644 dev-docs/changelog.adoc diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc new file mode 100644 index 000000000000..06b3fc3b51e5 --- /dev/null +++ b/dev-docs/changelog.adoc @@ -0,0 +1,116 @@ += Adding a Changelog Entry to Apache Solr +:toc: +:toclevels: 2 +:icons: font + +Since Solr 10.0 we no longer edit the monolithic *CHANGES.txt* file directly. +Instead we create a small YAML fragment **per bug-fix or feature**, checked into +`solr/changelog/unreleased-x.y.z/`. At build or release time these fragments are +automatically collated into a human-readable *CHANGES.md*. + +This document walks you through the workflow, explains the Gradle helpers +available, and finishes with a concrete example fragment. + +== 1. Workflow Overview + +. Make a feature branch with a descriptive name like 'SOLR-12345-fix-memory-leak'. +. Implement your change +. Run `./gradlew generateChangeYaml [-Pver=10.1.0]` +. Edit the generated YAML file to add a clear, user-focused description. +. Commit the YAML fragment along with the code change. +. Open your PR as usual. + +== 2. Directory Layout + +[source] +---- +solr/ +└── changelog/ + ├── unreleased/ ← new fragments live here + | └── SOLR-12345-fix-memory-leak.yml + ├── unreleased-10.1.0/ ← or in versioned folders + ├── v10.0.0/ ← generated at release time + └── v10.0.1/ +---- + +== 3. Gradle Tasks + +=== 3.1 Our Custom Task: `generateChangeYaml` + +Bootstraps a YAML file, by default in the `unreleased-x.y.z/` directory for +the current branch, or in a versioned sub-directory if `-Pver=10.1.0` is +specified. The task will use branch name as file name and title, and will +parse JIRA id from the branch name if it exists. + +It will use your git `user.name` as display name by default, but if you +add your name, github username, and optionally Apache ID to the +`gradle.properties` file at the root of the Solr git checkout, this task will +use that information. Example: + +[source, properties] +---- +# Changelog generator settings (./gradlew generateChangeYaml) +logchange.name=John Doe +logchange.nick=johngithub +logchange.asfid=johnapache +---- + +Invoke the task with: + +[source, bash] +---- +./gradlew generateChangeYaml [-Pver=10.1.0]` +---- + +=== .2 Plugin Tasks + +[cols="1,2,3", options="header"] +|=== +| Task | Purpose | Typical Usage + +| `logchangeGenerate` +| Generates changelog file () based on .yml entries and archives (does not moves any files) +| Doc generation, CI/CD + +| `logchangeRelease` +| Creates new changelog release by moving files from directory to vX.X.X directory +| Run by the RM during the release procedure. +|=== + + +== 4. Writing Good Entries + +* **Audience** is end-users and administrators, not committers. +* Keep the entry short and focused on the user impact. +* Choose the correct *type*: + ** `added` for new features + ** `fixed` for bug fixes + ** `deprecated` for deprecated features + ** `security` for security-related changes + ** `changed` for improvements + ** `deprecated` for deprecations + ** `removed` for code removed in major releases + ** `dependency_update` for updates to dependencies + ** `other` for anything else +* Reference issues as `SOLR-12345` or GitHub `#123`. + +== 5. Example YAML Fragment + +[source, yaml] +---- +title: Fix nasty bug in CoreContainer shutdown +# added/changed/deprecated/removed/fixed/security/dependency_update/other +type: fixed +authors: + - name: John Contributor + nick: johngithub + url: https://home.apache.org/phonebook.html?uid=johnasfid +links: + - name: SOLR-3333 + url: https://issues.apache.org/jira/browse/SOLR-3333 +---- + +== 6. Further Reading + +* <<_releasing_solr,Releasing Solr>> +* <> \ No newline at end of file diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index 827aa1fcec97..6dc4c1befe5e 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -108,3 +108,8 @@ tests.html=false # Disable auto JVM provisioning (we don't use toolchains yet but want no surprises). org.gradle.java.installations.auto-download=false + +# Changelog generator settings (./gradlew generateChangeYaml, see dev-docs/changelog.md) +#logchange.name= +#logchange.githubid= +#logchange.asfid= \ No newline at end of file From ebb0d821ee65ba0970cf9950ddd8677e90d84888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 02:44:13 +0200 Subject: [PATCH 08/64] Auto generate title Support PR ref More author details --- gradle/changelog.gradle | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 1b482741f169..e25dd093e6a2 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -16,33 +16,45 @@ */ logchange { - taskPrefix = "logchange" rootPath = "." inputDir = "changelog" unreleasedVersionDir = "unreleased" outputFile = "CHANGELOG.md" - configFile = "logchange-config.yml" generateChangesXml = false - xmlOutputFile = "changes.xml" } task generateChangeYaml { description = 'Generates a change/log description file (YAML)' doLast { def gitUserName = 'git config user.name'.execute().text.trim() + def configuredName = providers.gradleProperty("logchange.name").getOrElse(gitUserName) + def githubId = providers.gradleProperty("logchange.githubid").getOrElse(null) + def nick = githubId ? "\n nick: ${githubId}" : "" + def asfId = providers.gradleProperty("logchange.asfid").getOrElse(null) + def asfIdUrl = asfId ? "\n url: https://home.apache.org/phonebook.html?uid=${asfId}" : "" def gitBranch = 'git rev-parse --abbrev-ref HEAD'.execute().text.trim() def jiraMatcher = gitBranch =~ /SOLR-\d+/ def jiraRef = jiraMatcher ? jiraMatcher[0] : "SOLR-XXXX" - def fileName = "changelog/unreleased/${gitBranch}.yml" + def jiraUrl = "https://issues.apache.org/jira/browse/${jiraRef}" + def jiraLinks = jiraMatcher ? "links:\n - name: ${jiraRef}\n url: ${jiraUrl}" : "" + def githubMatcher = gitBranch =~ /(PR|GH|GITHUB|#)?(\d+)/ + def githubRef = githubMatcher ? githubMatcher.group(2) : "" + def githubLink = githubMatcher ? "issues:\n - ${githubRef}" : "" + def branchWithoutJira = gitBranch.replaceFirst(/SOLR-\d+-/, "").replaceFirst(/(PR|GH|GITHUB|#)?(\d+)/, "").replace("-", " ").capitalize() + def targetVer = providers.gradleProperty("ver").getOrElse(project.baseVersion) + if (!targetVer.matches("(9|1\\d)\\.\\d+\\.\\d+")) { + throw new GradleException("Invalid target version format: ${targetVer}. Expected format is x.y.z") + } + def fileName = "changelog/unreleased-${targetVer}/${gitBranch}.yml" def file = new File(fileName) file.parentFile.mkdirs() file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format +title: ${branchWithoutJira} type: other # added/changed/deprecated/removed/fixed/security/dependency_update/other -links: - - name: ${jiraRef} -title: TITLE HERE authors: - - name: ${gitUserName} + - name: ${configuredName}${nick}${asfIdUrl} +${jiraLinks} +${githubLink} """ println "Generated file: ${fileName} -- open it" From 03fb4555dc4e941868637bb76f936203fbde04d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 22:38:18 +0200 Subject: [PATCH 09/64] Prefix gradle.properties keys with 'user' --- dev-docs/changelog.adoc | 6 +++--- gradle/changelog.gradle | 6 +++--- gradle/template.gradle.properties | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 06b3fc3b51e5..a797064a564c 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -50,9 +50,9 @@ use that information. Example: [source, properties] ---- # Changelog generator settings (./gradlew generateChangeYaml) -logchange.name=John Doe -logchange.nick=johngithub -logchange.asfid=johnapache +user.name=John Doe +user.githubid=johngithub +user.asfid=johnapache ---- Invoke the task with: diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index e25dd093e6a2..039140581ef4 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -27,10 +27,10 @@ task generateChangeYaml { description = 'Generates a change/log description file (YAML)' doLast { def gitUserName = 'git config user.name'.execute().text.trim() - def configuredName = providers.gradleProperty("logchange.name").getOrElse(gitUserName) - def githubId = providers.gradleProperty("logchange.githubid").getOrElse(null) + def configuredName = providers.gradleProperty("user.name").getOrElse(gitUserName) + def githubId = providers.gradleProperty("user.githubid").getOrElse(null) def nick = githubId ? "\n nick: ${githubId}" : "" - def asfId = providers.gradleProperty("logchange.asfid").getOrElse(null) + def asfId = providers.gradleProperty("user.asfid").getOrElse(null) def asfIdUrl = asfId ? "\n url: https://home.apache.org/phonebook.html?uid=${asfId}" : "" def gitBranch = 'git rev-parse --abbrev-ref HEAD'.execute().text.trim() def jiraMatcher = gitBranch =~ /SOLR-\d+/ diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index e0a6599aa7af..e2236d78e13a 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -126,6 +126,6 @@ production=false disableUiModule=false # Changelog generator settings (./gradlew generateChangeYaml, see dev-docs/changelog.md) -#logchange.name= -#logchange.githubid= -#logchange.asfid= \ No newline at end of file +#user.name= +#user.githubid= +#user.asfid= \ No newline at end of file From 09756e12f5b067d4c4865d8089089dee673707d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 22:38:31 +0200 Subject: [PATCH 10/64] Update dev-docs/changelog.adoc Co-authored-by: David Smiley --- dev-docs/changelog.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 06b3fc3b51e5..6a10d05332a9 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -91,7 +91,7 @@ Invoke the task with: ** `deprecated` for deprecations ** `removed` for code removed in major releases ** `dependency_update` for updates to dependencies - ** `other` for anything else + ** `other` for anything else, like the build or documentation * Reference issues as `SOLR-12345` or GitHub `#123`. == 5. Example YAML Fragment From 26be74c8361c8c45311d115362c7ef109a8a024b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sat, 21 Jun 2025 22:39:03 +0200 Subject: [PATCH 11/64] Repeated 'deprecated' docs --- dev-docs/changelog.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index a797064a564c..8f37029c2ea9 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -88,7 +88,6 @@ Invoke the task with: ** `deprecated` for deprecated features ** `security` for security-related changes ** `changed` for improvements - ** `deprecated` for deprecations ** `removed` for code removed in major releases ** `dependency_update` for updates to dependencies ** `other` for anything else From 1f1be031940224da4e6aa18ae35a4f3e4bbf1ca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sun, 22 Jun 2025 15:45:40 +0200 Subject: [PATCH 12/64] Upgrade to 1.19.5 and start using versioned unreleased dirs Add CHANGELOG.md to .gitignore --- .gitignore | 1 + changelog/{unreleased => unreleased-10.0.0}/.keep | 0 gradle/libs.versions.toml | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) rename changelog/{unreleased => unreleased-10.0.0}/.keep (100%) diff --git a/.gitignore b/.gitignore index 42f1ceec7f3e..4cc9f744f99e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /gradle.properties .gradle/ build/ +CHANGELOG.md # IntelliJ IDEA /.idea/ diff --git a/changelog/unreleased/.keep b/changelog/unreleased-10.0.0/.keep similarity index 100% rename from changelog/unreleased/.keep rename to changelog/unreleased-10.0.0/.keep diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 551cd1d2b0ba..2749caa1316f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -156,7 +156,7 @@ langchain4j = "0.35.0" link-checker = "1.4.2" littlerobots-versioncatalogupdate = "0.8.4" lmax-disruptor = "3.4.4" -logchange = "1.19.4" +logchange = "1.19.5" ltgt-errorprone = "3.1.0" mockito = "5.16.1" morethan-jmhreport = "0.9.0" From 9d57c4948290aa864de89c3be70d0219442bc21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sun, 22 Jun 2025 15:48:43 +0200 Subject: [PATCH 13/64] Remove config for unreleasedVersionDir --- changelog/logchange-config.yml | 1 - gradle/changelog.gradle | 1 - 2 files changed, 2 deletions(-) diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 762152b99eb9..73b8aa83af1b 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -18,7 +18,6 @@ # More info about configuration you can find https://github.com/logchange/logchange#configuration changelog: heading: Some information that will be display in the top of CHANGELOG.md - unreleasedVersionDir: unreleased # You can define custom entry types, which overriders default ones # [added/changed/deprecated/removed/fixed/security/dependency_update/other] # Remember to adjust labels below, to match new entry types f.e. add: New features diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 039140581ef4..35601106d5a6 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -18,7 +18,6 @@ logchange { rootPath = "." inputDir = "changelog" - unreleasedVersionDir = "unreleased" outputFile = "CHANGELOG.md" generateChangesXml = false } From 68343bf898ad21fa8627b43ea86b66ae443aa845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sun, 22 Jun 2025 21:40:40 +0200 Subject: [PATCH 14/64] Customized entryTypes for solr Kept all the default keys and added 'optimized'. Used header texts that the project has used before --- changelog/logchange-config.yml | 52 +++++++++++++++++----------------- dev-docs/changelog.adoc | 7 +++-- gradle/changelog.gradle | 2 +- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 73b8aa83af1b..02b92574dfcb 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -18,42 +18,42 @@ # More info about configuration you can find https://github.com/logchange/logchange#configuration changelog: heading: Some information that will be display in the top of CHANGELOG.md - # You can define custom entry types, which overriders default ones - # [added/changed/deprecated/removed/fixed/security/dependency_update/other] - # Remember to adjust labels below, to match new entry types f.e. add: New features - # entryTypes: # you can define custom entry types - # - key: add - # order: 1 - # - key: fix - # order: 2 - # - key: change - # order: 3 - # - key: other - # order: 4 + entryTypes: # you can define custom entry types + - key: added + order: 1 + - key: changed + order: 2 + - key: optimized + order: 3 + - key: fixed + order: 4 + - key: deprecated + order: 5 + - key: removed + order: 6 + - key: dependency_update + order: 7 + - key: security + order: 8 + - key: other + order: 9 labels: unreleased: unreleased important_notes: Important notes types: entryTypesLabels: + added: New Features + changed: Improvements + optimized: Optimizations + fixed: Bug Fixes + deprecated: Deprecations + removed: Removed + dependency_update: Dependency Upgrades security: Security other: Other - removed: Removed - added: Added - deprecated: Deprecated - fixed: Fixed - dependency_update: Dependency updates - changed: Changed number_of_changes: singular: change plural: changes - security: Security - other: Other - removed: Removed - added: Added - deprecated: Deprecated - fixed: Fixed - dependency_update: Dependency updates - changed: Changed configuration: heading: Configuration changes type: Type diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 07bc093a9d86..34890a880772 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -84,12 +84,13 @@ Invoke the task with: * Keep the entry short and focused on the user impact. * Choose the correct *type*: ** `added` for new features + ** `changed` for improvements + ** `optimized` for optimizations (performance, memory, etc.) ** `fixed` for bug fixes ** `deprecated` for deprecated features - ** `security` for security-related changes - ** `changed` for improvements ** `removed` for code removed in major releases ** `dependency_update` for updates to dependencies + ** `security` for security-related changes ** `other` for anything else, like the build or documentation * Reference issues as `SOLR-12345` or GitHub `#123`. @@ -98,7 +99,7 @@ Invoke the task with: [source, yaml] ---- title: Fix nasty bug in CoreContainer shutdown -# added/changed/deprecated/removed/fixed/security/dependency_update/other +# added, changed, optimized, fixed, deprecated, removed, dependency_update, security, other type: fixed authors: - name: John Contributor diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 35601106d5a6..863748740066 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -49,7 +49,7 @@ task generateChangeYaml { file.parentFile.mkdirs() file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format title: ${branchWithoutJira} -type: other # added/changed/deprecated/removed/fixed/security/dependency_update/other +type: other # added, changed, optimized, fixed, deprecated, removed, dependency_update, security, other authors: - name: ${configuredName}${nick}${asfIdUrl} ${jiraLinks} From 362acdfe5cf6bcb59c4d8b3884f5eaf2b452f0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 20:58:33 +0200 Subject: [PATCH 15/64] Use single unreleased folder --- changelog/{unreleased-10.0.0 => unreleased}/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased-10.0.0 => unreleased}/.keep (100%) diff --git a/changelog/unreleased-10.0.0/.keep b/changelog/unreleased/.keep similarity index 100% rename from changelog/unreleased-10.0.0/.keep rename to changelog/unreleased/.keep From cc696a02f34a25448b2d1354cbe7218b0013d5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 21:08:57 +0200 Subject: [PATCH 16/64] Do not gitignore CHANGELOG.md --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4cc9f744f99e..05199687470f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ /gradle.properties .gradle/ build/ -CHANGELOG.md # IntelliJ IDEA /.idea/ @@ -42,4 +41,4 @@ gradle/wrapper/gradle-wrapper.jar # WANT TO ADD MORE? You can tell Git without adding to this file: # See https://git-scm.com/docs/gitignore -# In particular, if you have tools you use, add to $GIT_DIR/info/exclude or use core.excludesFile \ No newline at end of file +# In particular, if you have tools you use, add to $GIT_DIR/info/exclude or use core.excludesFile From 0c1fa197729bd2954e7faca6fd13afb1bdb004ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 21:18:36 +0200 Subject: [PATCH 17/64] Remove target version feature from changelog.gradle --- gradle/changelog.gradle | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 863748740066..9085a7679c4c 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -40,11 +40,7 @@ task generateChangeYaml { def githubRef = githubMatcher ? githubMatcher.group(2) : "" def githubLink = githubMatcher ? "issues:\n - ${githubRef}" : "" def branchWithoutJira = gitBranch.replaceFirst(/SOLR-\d+-/, "").replaceFirst(/(PR|GH|GITHUB|#)?(\d+)/, "").replace("-", " ").capitalize() - def targetVer = providers.gradleProperty("ver").getOrElse(project.baseVersion) - if (!targetVer.matches("(9|1\\d)\\.\\d+\\.\\d+")) { - throw new GradleException("Invalid target version format: ${targetVer}. Expected format is x.y.z") - } - def fileName = "changelog/unreleased-${targetVer}/${gitBranch}.yml" + def fileName = "changelog/unreleased/${gitBranch}.yml" def file = new File(fileName) file.parentFile.mkdirs() file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format @@ -58,4 +54,4 @@ ${githubLink} println "Generated file: ${fileName} -- open it" } -} \ No newline at end of file +} From e890f164697f8bd6f8d2a58f061ddf1eed0dc65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 21:18:54 +0200 Subject: [PATCH 18/64] Rewrite dev-docs for single unreleased folder --- dev-docs/changelog.adoc | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 34890a880772..69e9008ff79e 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -3,10 +3,10 @@ :toclevels: 2 :icons: font -Since Solr 10.0 we no longer edit the monolithic *CHANGES.txt* file directly. -Instead we create a small YAML fragment **per bug-fix or feature**, checked into -`solr/changelog/unreleased-x.y.z/`. At build or release time these fragments are -automatically collated into a human-readable *CHANGES.md*. +Since Solr 10.0 we no longer edit a monolithic *CHANGES.txt* file directly. +Instead, we create a small YAML fragment **per bug-fix or feature**, checked into +`solr/changelog/unreleased/`. At release time these fragments are automatically +collated into a human-readable *CHANGES.md*. This document walks you through the workflow, explains the Gradle helpers available, and finishes with a concrete example fragment. @@ -15,8 +15,8 @@ available, and finishes with a concrete example fragment. . Make a feature branch with a descriptive name like 'SOLR-12345-fix-memory-leak'. . Implement your change -. Run `./gradlew generateChangeYaml [-Pver=10.1.0]` -. Edit the generated YAML file to add a clear, user-focused description. +. Run `./gradlew generateChangeYaml` +. Edit the generated YAML file in `changelog/unreleased/` to add a clear, user-focused description. . Commit the YAML fragment along with the code change. . Open your PR as usual. @@ -28,8 +28,7 @@ solr/ └── changelog/ ├── unreleased/ ← new fragments live here | └── SOLR-12345-fix-memory-leak.yml - ├── unreleased-10.1.0/ ← or in versioned folders - ├── v10.0.0/ ← generated at release time + ├── v10.0.0/ ← changes in already released version └── v10.0.1/ ---- @@ -37,13 +36,12 @@ solr/ === 3.1 Our Custom Task: `generateChangeYaml` -Bootstraps a YAML file, by default in the `unreleased-x.y.z/` directory for -the current branch, or in a versioned sub-directory if `-Pver=10.1.0` is -specified. The task will use branch name as file name and title, and will +Bootstraps a YAML file, by default in the `changelog/unreleased/` directory for +the current branch, The task will use branch name as file name and title, and will parse JIRA id from the branch name if it exists. It will use your git `user.name` as display name by default, but if you -add your name, github username, and optionally Apache ID to the +add your name, GitHub username, and optionally Apache ID to the `gradle.properties` file at the root of the Solr git checkout, this task will use that information. Example: @@ -59,10 +57,10 @@ Invoke the task with: [source, bash] ---- -./gradlew generateChangeYaml [-Pver=10.1.0]` +./gradlew generateChangeYaml ---- -=== .2 Plugin Tasks +=== 3.2 Plugin Tasks [cols="1,2,3", options="header"] |=== @@ -73,11 +71,10 @@ Invoke the task with: | Doc generation, CI/CD | `logchangeRelease` -| Creates new changelog release by moving files from directory to vX.X.X directory -| Run by the RM during the release procedure. +| Creates new changelog release by moving files from directory to vX.Y.Z directory +| Run by the Release Manager during the release procedure. |=== - == 4. Writing Good Entries * **Audience** is end-users and administrators, not committers. @@ -112,5 +109,5 @@ links: == 6. Further Reading -* <<_releasing_solr,Releasing Solr>> -* <> \ No newline at end of file +* xref:https://github.com/logchange/logchange[Logchange web page] +* xref:https://keepachangelog.com/en/1.1.0/[keepachangelog.com website] From 8337c29b7f209506a67f709310397e11bac0fbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 21:19:27 +0200 Subject: [PATCH 19/64] Bump logchange version --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ffcb8f33bbad..c278e2bfc087 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -162,7 +162,7 @@ langchain4j = "0.35.0" link-checker = "1.4.2" littlerobots-versioncatalogupdate = "0.8.5" lmax-disruptor = "3.4.4" -logchange = "1.19.5" +logchange = "1.19.10" ltgt-errorprone = "3.1.0" mockito = "5.19.0" morethan-jmhreport = "0.9.6" From 0faad7c6baa7d7e7d679c91f91910047e25a099a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Mon, 20 Oct 2025 22:00:32 +0200 Subject: [PATCH 20/64] New script changes2logchange.py to migrate CHANGES.txt to logchange folders --- changelog/logchange-config.yml | 23 +- dev-docs/changelog.adoc | 1 + dev-tools/scripts/README.md | 79 ++- dev-tools/scripts/changes2logchange.py | 744 +++++++++++++++++++++++++ 4 files changed, 809 insertions(+), 38 deletions(-) create mode 100755 dev-tools/scripts/changes2logchange.py diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 02b92574dfcb..651f609f0b91 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -17,26 +17,28 @@ # More info about configuration you can find https://github.com/logchange/logchange#configuration changelog: - heading: Some information that will be display in the top of CHANGELOG.md + heading: This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc entryTypes: # you can define custom entry types - - key: added + - key: upgrade_notes order: 1 - - key: changed + - key: added order: 2 - - key: optimized + - key: changed order: 3 - - key: fixed + - key: optimized order: 4 - - key: deprecated + - key: fixed order: 5 - - key: removed + - key: deprecated order: 6 - - key: dependency_update + - key: removed order: 7 - - key: security + - key: dependency_update order: 8 - - key: other + - key: security order: 9 + - key: other + order: 10 labels: unreleased: unreleased important_notes: Important notes @@ -51,6 +53,7 @@ changelog: dependency_update: Dependency Upgrades security: Security other: Other + upgrade_notes: Upgrade Notes number_of_changes: singular: change plural: changes diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 69e9008ff79e..7ff3ebf32583 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -89,6 +89,7 @@ Invoke the task with: ** `dependency_update` for updates to dependencies ** `security` for security-related changes ** `other` for anything else, like the build or documentation + ** `upgrade_notes` for upgrade notes, only used in older versions * Reference issues as `SOLR-12345` or GitHub `#123`. == 5. Example YAML Fragment diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.md index fc8074fd3da6..0681b069d02a 100644 --- a/dev-tools/scripts/README.md +++ b/dev-tools/scripts/README.md @@ -18,11 +18,11 @@ the full tests. usage: smokeTestRelease.py [-h] [--tmp-dir PATH] [--not-signed] [--local-keys PATH] [--revision REVISION] [--version X.Y.Z(-ALPHA|-BETA)?] [--test-alt-java TEST_ALT_JAVA] [--download-only] [--dev-mode] url ... Utility to test a release. - + positional arguments: url Url pointing to release to test test_args Arguments to pass to gradle for testing, e.g. -Dwhat=ever. - + options: -h, --help show this help message and exit --tmp-dir PATH Temporary directory to test inside, defaults to /tmp/smoke_solr_$version_$revision @@ -35,32 +35,32 @@ the full tests. Path to Java alternative home directory, to run tests with if specified --download-only Only perform download and sha hash check steps --dev-mode Enable dev mode, will not check branch compatibility - + Example usage: python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/solr/solr-10.0.0-RC1-rev-c7510a0... ### releaseWizard.py -The Release Wizard guides the Release Manager through the release process step +The Release Wizard guides the Release Manager through the release process step by step, helping you to to run the right commands in the right order, generating e-mail templates with the correct texts, versions, paths etc, obeying the voting rules and much more. It also serves as a documentation of all the steps, with timestamps, preserving log files from each command etc, showing only the steps and commands required for a major/minor/bugfix release. It also lets -you generate a full Asciidoc guide for the release. The wizard will execute many -of the other tools in this folder. +you generate a full Asciidoc guide for the release. The wizard will execute many +of the other tools in this folder. usage: releaseWizard.py [-h] [--dry-run] [--init] - + Script to guide a RM through the whole release process - + optional arguments: -h, --help show this help message and exit --dry-run Do not execute any commands, but echo them instead. Display extra debug info --init Re-initialize root and version - + Go push that release! ### buildAndPushRelease.py @@ -68,9 +68,9 @@ of the other tools in this folder. usage: buildAndPushRelease.py [-h] [--no-prepare] [--local-keys PATH] [--push-local PATH] [--sign FINGERPRINT] [--sign-method-gradle] [--gpg-pass-noprompt] [--gpg-home PATH] [--rc-num NUM] [--root PATH] [--logfile PATH] [--dev-mode] - + Utility to build, push, and test a release. - + optional arguments: -h, --help show this help message and exit --no-prepare Use the already built release in the provided checkout @@ -89,19 +89,19 @@ of the other tools in this folder. --root PATH Root of Git working tree for solr. Default: "." (the current directory) --logfile PATH Specify log file path (default /tmp/release.log) --dev-mode Enable development mode, which disables some strict checks - + Example usage for a Release Manager: python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 3782CBB60147010B330523DD26FBCC7836BF353A --rc-num 1 ### addVersion.py usage: addVersion.py [-h] [-l LUCENE_VERSION] version - + Add a new version to CHANGES, to Version.java, build.gradle and solrconfig.xml files - + positional arguments: version New Solr version - + optional arguments: -h, --help show this help message and exit -l LUCENE_VERSION Optional lucene version. By default will read gradle/libs.versions.toml @@ -113,21 +113,21 @@ under the given version in the given CHANGES.txt file and prints a regular expression that will match all of them usage: releasedJirasRegex.py [-h] version changes - + Prints a regex matching JIRAs fixed in the given version by parsing the given CHANGES.txt file - + positional arguments: version Version of the form X.Y.Z changes CHANGES.txt file to parse - + optional arguments: -h, --help show this help message and exit ### reproduceJenkinsFailures.py usage: reproduceJenkinsFailures.py [-h] [--no-git] [--iters N] URL - + Must be run from a Solr git workspace. Downloads the Jenkins log pointed to by the given URL, parses it for Git revision and failed Solr tests, checks out the Git revision in the local workspace, @@ -136,10 +136,10 @@ and prints a regular expression that will match all of them in each module of interest, failing at the end if any of the runs fails. To control the maximum number of concurrent JVMs used for each module's test run, set 'tests.jvms', e.g. in ~/lucene.build.properties - + positional arguments: URL Points to the Jenkins log to parse - + optional arguments: -h, --help show this help message and exit --no-git Do not run "git" at all @@ -148,9 +148,9 @@ and prints a regular expression that will match all of them ### githubPRs.py usage: githubPRs.py [-h] [--json] [--token TOKEN] - + Find open Pull Requests that need attention - + optional arguments: -h, --help show this help message and exit --json Output as json @@ -162,19 +162,42 @@ Scaffold a new module and include it into the build. It will set up the folders and all for you, so the only thing you need to do is add classes, tests and test-data. usage: scaffoldNewModule.py [-h] name full_name description - + Scaffold new module into solr/modules/ - + positional arguments: name code-name/id, e.g. my-module full_name Readable name, e.g. "My Module" description Short description for docs - + optional arguments: -h, --help show this help message and exit Example: ./scaffoldNewModule.py foo "My Module" "Very Useful module here" +### changes2logchange.py + +Migrates the legacy CHANGES.txt file format to the new logchange YAML-based format. +This script parses the monolithic CHANGES.txt file and generates individual YAML +files for each changelog entry, organized by version (v10.0.0/, v9.9.0/, etc.). + +Each YAML file complies with the schema outlined in `dev-docs/changelog.adoc`. + + usage: changes2logchange.py [] + + Positional arguments: + CHANGES.txt Path to the CHANGES.txt file to migrate + output_dir Directory to write changelog/ structure (default: ./changelog) + + Example usage: + python3 dev-tools/scripts/changes2logchange.py solr/CHANGES.txt changelog/ + + Output: + - Creates vX.Y.Z/ directories for each version + - Generates SOLR-XXXXX-slug-name.yml files for entries with JIRA issues + - Skips entries without traceable issue/PR references + - Reports migration statistics (versions, entries migrated, entries skipped) + ### gitignore-gen.sh TBD @@ -191,11 +214,11 @@ TBD -r Specify remote to push to. Defaults to 'origin' -p Push to remote. Only done if both cherry-pick and tests succeeded WARNING: Never push changes to a remote branch before a thorough local test - + Simple script for aiding in back-porting one or more (trivial) commits to other branches. On merge conflict the script will run 'git mergetool'. See 'git mergetool --help' for help on configuring your favourite merge tool. Check out Sublime Merge (smerge). - + Example: # Backport two commits to both stable and release branches dev-tools/scripts/cherrypick.sh -b branch_9x -b branch_9_0 deadbeef0000 cafebabe1111 diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py new file mode 100755 index 000000000000..5d94c45731db --- /dev/null +++ b/dev-tools/scripts/changes2logchange.py @@ -0,0 +1,744 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Migration script to convert Apache Solr's legacy CHANGES.txt file format +to the new logchange YAML-based format. + +This script parses the monolithic CHANGES.txt file and generates individual +YAML files for each changelog entry, organized by version. +""" + +import os +import re +import sys +import json +import yaml +import html +from pathlib import Path +from dataclasses import dataclass, asdict, field +from typing import List, Optional, Tuple + + +class ChangeType: + """Mapping of CHANGES.txt section headings to logchange types.""" + + # Section headings that should be skipped entirely (no entries created) + SKIP_SECTIONS = { + "Versions of Major Components", + "Detailed Change List", + "Upgrading from Solr any prior release", + "Upgrading from previous Solr versions", + "System Requirements", + "Lucene Information", + "Status", + } + + # Maps various section heading patterns to logchange types + HEADING_MAP = { + # New Features / Additions + "New Features": "added", + "Features": "added", + "New Functionality": "added", + + # Improvements / Changes + "Improvements": "changed", + "Enhancements": "changed", + "Changes": "changed", + "Improvements / Changes": "changed", + + # Performance / Optimizations + "Optimizations": "optimized", + "Performance": "optimized", + "Optimization": "optimized", + + # Bug Fixes + "Bug Fixes": "fixed", + "Bug Fix": "fixed", + "Bugs": "fixed", + + # Deprecations + "Deprecations": "deprecated", + "Deprecation": "deprecated", + "Deprecation Notices": "deprecated", + "Deprecation Removals": "removed", # This is more about removals but was in Deprecations section + + # Removed / Removed Features + "Removed": "removed", + "Removal": "removed", + "Removed Features": "removed", + "Removals": "removed", + + # Security + "Security": "security", + "Security Fixes": "security", + + # Dependency Upgrades + "Dependency Upgrades": "dependency_update", + "Dependency Updates": "dependency_update", + "Dependency Upgrade": "dependency_update", + "Dependencies": "dependency_update", + + # Build / Infrastructure + "Build": "other", + "Build Changes": "other", + "Build Fixes": "other", + + # Upgrade Notes - special category + "Upgrade Notes": "upgrade_notes", + + # Sections to skip (no migration) + # These are handled specially and should not create entries + # "Versions of Major Components": SKIP, + # "Detailed Change List": SKIP, + # etc. + + # Other + "Other Changes": "other", + "Other": "other", + "Miscellaneous": "other", + "Docker": "other", + "Ref Guide": "other", + "Documentation": "other", + } + + @staticmethod + def get_type(heading: str) -> str: + """Map a section heading to a logchange type.""" + heading_normalized = heading.strip() + if heading_normalized in ChangeType.HEADING_MAP: + return ChangeType.HEADING_MAP[heading_normalized] + + # Fallback: try case-insensitive matching + for key, value in ChangeType.HEADING_MAP.items(): + if key.lower() == heading_normalized.lower(): + return value + + # Default to "other" if no match found + print(f"Warning: Unknown section heading '{heading}', defaulting to 'other'", file=sys.stderr) + return "other" + + +@dataclass +class Author: + """Represents a changelog entry author/contributor.""" + name: str + nick: Optional[str] = None + url: Optional[str] = None + + def to_dict(self): + """Convert to dictionary, excluding None values.""" + result = {"name": self.name} + if self.nick: + result["nick"] = self.nick + if self.url: + result["url"] = self.url + return result + + +@dataclass +class Link: + """Represents a link (JIRA issue or GitHub PR).""" + name: str + url: str + + def to_dict(self): + """Convert to dictionary.""" + return {"name": self.name, "url": self.url} + + +@dataclass +class ChangeEntry: + """Represents a single changelog entry.""" + title: str + change_type: str + authors: List[Author] = field(default_factory=list) + links: List[Link] = field(default_factory=list) + + def to_dict(self): + """Convert to dictionary for YAML serialization.""" + return { + "title": self.title, + "type": self.change_type, + "authors": [author.to_dict() for author in self.authors], + "links": [link.to_dict() for link in self.links], + } + + +class AuthorParser: + """Parses author/contributor information from entry text.""" + + # Pattern to match TRAILING author list at the end of an entry: (Author1, Author2 via Committer) + # Must be at the very end, possibly with trailing punctuation + # Strategy: Match from the last '(' that leads to end-of-string pattern matching + # This regex finds the LAST occurrence of a parenthesized group followed by optional whitespace/punctuation + # and then end of string + AUTHOR_PATTERN = re.compile(r'\s+\(([^()]+)\)\s*[.,]?\s*$', re.MULTILINE) + + @staticmethod + def parse_authors(entry_text: str) -> Tuple[str, List[Author]]: + """ + Extract authors from entry text. + + Returns: + Tuple of (cleaned_text, list_of_authors) + + Patterns handled: + - (Author Name) + - (Author1, Author2) + - (Author Name via CommitterName) + - (Author1 via Committer1, Author2 via Committer2) + + Only matches author attribution at the END of the entry text, + not in the middle of descriptions like (aka Standalone) + """ + # Find ALL matches and use the LAST one (rightmost) + # This ensures we get the actual author attribution, not mid-text parentheses + matches = list(AuthorParser.AUTHOR_PATTERN.finditer(entry_text)) + if not matches: + return entry_text, [] + + # Use the last match (rightmost) + match = matches[-1] + + author_text = match.group(1) + # Include the space before the parenthesis in what we remove + cleaned_text = entry_text[:match.start()].rstrip() + + authors = [] + + # Split by comma, but be aware of "via" keyword + # Pattern: "Author via Committer" or just "Author" + segments = [seg.strip() for seg in author_text.split(',')] + + for segment in segments: + segment = segment.strip() + if not segment: + continue + + # Handle "via" prefix (standalone or after author name) + if segment.startswith('via '): + # Malformed: standalone "via Committer" (comma was added incorrectly) + # Extract just the committer name + committer_name = segment[4:].strip() # Remove "via " prefix + if committer_name: + authors.append(Author(name=committer_name)) + elif ' via ' in segment: + # Format: "Author via Committer" + parts = segment.split(' via ') + author_name = parts[0].strip() + + if author_name: + # Normal case: "Author via Committer" - add the author + authors.append(Author(name=author_name)) + else: + # Should not happen, but handle it + committer_name = parts[1].strip() if len(parts) > 1 else "" + if committer_name: + authors.append(Author(name=committer_name)) + else: + # Just an author name + authors.append(Author(name=segment)) + + return cleaned_text, authors + + +class IssueExtractor: + """Extracts issue/PR references from entry text.""" + + JIRA_ISSUE_PATTERN = re.compile(r'(?:SOLR|LUCENE|INFRA)-(\d+)') + GITHUB_PR_PATTERN = re.compile(r'(?:GitHub\s*)?#(\d+)') + + @staticmethod + def extract_issues(entry_text: str) -> List[Link]: + """Extract JIRA and GitHub issue references.""" + links = [] + seen_issues = set() # Track seen issues to avoid duplicates + + # Extract SOLR, LUCENE, INFRA issues + for match in IssueExtractor.JIRA_ISSUE_PATTERN.finditer(entry_text): + issue_id = match.group(0) # Full "SOLR-12345" or "LUCENE-12345" format + if issue_id not in seen_issues: + url = f"https://issues.apache.org/jira/browse/{issue_id}" + links.append(Link(name=issue_id, url=url)) + seen_issues.add(issue_id) + + # Extract GitHub PRs in multiple formats: + # "PR#3758", "PR-2475", "GITHUB#3666" + github_patterns = [ + (r'PR[#-](\d+)', 'PR#'), # PR#1234 or PR-1234 + (r'GITHUB#(\d+)', 'GITHUB#'), # GITHUB#3666 + ] + + for pattern_str, prefix in github_patterns: + pattern = re.compile(pattern_str) + for match in pattern.finditer(entry_text): + pr_num = match.group(1) + pr_name = f"{prefix}{pr_num}" + if pr_name not in seen_issues: + url = f"https://github.com/apache/solr/pull/{pr_num}" + links.append(Link(name=pr_name, url=url)) + seen_issues.add(pr_name) + + return links + + +class SlugGenerator: + """Generates slug-style filenames for YAML files.""" + + # Characters that are unsafe in filenames on various filesystems + # Avoid: < > : " / \ | ? * and control characters + # Note: # is safe on most filesystems + # Also avoid multiple consecutive dashes + UNSAFE_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*\x00-\x1f]+') + + @staticmethod + def generate_slug(issue_id: str, title: str) -> str: + """ + Generate a slug from issue ID and title. + + Format: ISSUE-12345-short-slug or VERSION-entry-001-short-slug + Uses the actual issue ID without forcing SOLR- prefix + Ensures filesystem-safe filenames and respects word boundaries + """ + # Sanitize issue_id to remove unsafe characters (preserve case and # for readability) + base_issue = SlugGenerator._sanitize_issue_id(issue_id) + + # Create slug from title: lowercase, replace unsafe chars with dash + title_slug = SlugGenerator._sanitize_filename_part(title) + + # Limit to reasonable length while respecting word boundaries + # Target max length: 50 chars for slug (leaving room for base_issue and dash) + if len(title_slug) > 50: + # Find last word boundary within 50 chars + truncated = title_slug[:50] + # Find the last dash (word boundary) + last_dash = truncated.rfind('-') + if last_dash > 20: # Keep at least 20 chars to avoid too-short slugs + title_slug = truncated[:last_dash] + else: + # If no good word boundary found, use hard limit + title_slug = truncated.rstrip('-') + + return f"{base_issue}-{title_slug}" + + @staticmethod + def _sanitize_issue_id(issue_id: str) -> str: + """ + Sanitize issue ID while preserving uppercase letters and # for readability. + Examples: SOLR-12345, LUCENE-1234, PR#3758, GITHUB#2408, v9.8.0-entry-001 + """ + # Replace unsafe characters with dash (preserving case) + sanitized = SlugGenerator.UNSAFE_CHARS_PATTERN.sub('-', issue_id) + + # Replace remaining unsafe characters (but keep letters/numbers/dash/hash/dot) + sanitized = re.sub(r'[^a-zA-Z0-9.#-]+', '-', sanitized) + + # Replace multiple consecutive dashes with single dash + sanitized = re.sub(r'-+', '-', sanitized) + + # Strip leading/trailing dashes + sanitized = sanitized.strip('-') + + return sanitized + + @staticmethod + def _sanitize_filename_part(text: str) -> str: + """ + Sanitize text for use in filenames. + - Convert to lowercase + - Replace unsafe characters with dashes + - Remove multiple consecutive dashes + - Strip leading/trailing dashes + """ + # Convert to lowercase + text = text.lower() + + # Replace unsafe characters with dash + text = SlugGenerator.UNSAFE_CHARS_PATTERN.sub('-', text) + + # Replace non-alphanumeric (except dash) with dash + text = re.sub(r'[^a-z0-9-]+', '-', text) + + # Replace multiple consecutive dashes with single dash + text = re.sub(r'-+', '-', text) + + # Strip leading/trailing dashes + text = text.strip('-') + + return text + + +class VersionSection: + """Represents all entries for a specific version.""" + + def __init__(self, version: str): + self.version = version + self.entries: List[ChangeEntry] = [] + + def add_entry(self, entry: ChangeEntry): + """Add an entry to this version.""" + self.entries.append(entry) + + def get_directory_name(self) -> str: + """Get the directory name for this version (e.g., 'v10.0.0').""" + return f"v{self.version}" + + +class ChangesParser: + """Main parser for CHANGES.txt file.""" + + # Pattern to match version headers: ================== 10.0.0 ================== + VERSION_HEADER_PATTERN = re.compile(r'=+\s+([\d.]+)\s+=+') + + # Pattern to match section headers: "Section Name" followed by dashes + # Matches patterns like "New Features\n---------------------" + SECTION_HEADER_PATTERN = re.compile(r'^([A-Za-z][A-Za-z0-9\s/&-]*?)\n\s*-+\s*$', re.MULTILINE) + + def __init__(self, changes_file_path: str): + self.changes_file_path = changes_file_path + self.versions: List[VersionSection] = [] + + def parse(self): + """Parse the CHANGES.txt file.""" + with open(self.changes_file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Split into version sections + version_matches = list(self.VERSION_HEADER_PATTERN.finditer(content)) + + for i, version_match in enumerate(version_matches): + version = version_match.group(1) + start_pos = version_match.end() + + # Find the end of this version section (start of next version or EOF) + if i + 1 < len(version_matches): + end_pos = version_matches[i + 1].start() + else: + end_pos = len(content) + + version_content = content[start_pos:end_pos] + version_section = self._parse_version_section(version, version_content) + self.versions.append(version_section) + + def _parse_version_section(self, version: str, content: str) -> VersionSection: + """Parse all entries within a single version section.""" + version_section = VersionSection(version) + + # Split into subsections (New Features, Bug Fixes, etc.) + section_matches = list(self.SECTION_HEADER_PATTERN.finditer(content)) + + for i, section_match in enumerate(section_matches): + section_name = section_match.group(1) + + # Skip sections that should not be migrated + if section_name in ChangeType.SKIP_SECTIONS: + continue + + section_type = ChangeType.get_type(section_name) + + start_pos = section_match.end() + + # Find the end of this section (start of next section or EOF) + if i + 1 < len(section_matches): + end_pos = section_matches[i + 1].start() + else: + end_pos = len(content) + + section_content = content[start_pos:end_pos] + + # Parse entries in this section + entries = self._parse_entries(section_content, section_type) + for entry in entries: + version_section.add_entry(entry) + + return version_section + + def _parse_entries(self, section_content: str, change_type: str) -> List[ChangeEntry]: + """Parse individual entries within a section. + + Handles both: + - Bulleted entries: * text + - Numbered entries: 1. text, 2. text, etc. (older format) + """ + entries = [] + + # First try to split by bulleted entries (* prefix) + bulleted_pattern = re.compile(r'^\*\s+', re.MULTILINE) + bulleted_entries = bulleted_pattern.split(section_content) + + if len(bulleted_entries) > 1: + # Has bulleted entries + for entry_text in bulleted_entries[1:]: # Skip first empty split + entry_text = entry_text.strip() + if not entry_text or entry_text == "(No changes)": + continue + entry = self._parse_single_entry(entry_text, change_type) + if entry: + entries.append(entry) + else: + # No bulleted entries, try numbered entries (old format: "1. text", "2. text", etc.) + numbered_pattern = re.compile(r'^\s{0,2}\d+\.\s+', re.MULTILINE) + if numbered_pattern.search(section_content): + # Has numbered entries + numbered_entries = numbered_pattern.split(section_content) + for entry_text in numbered_entries[1:]: # Skip first empty split + entry_text = entry_text.strip() + if not entry_text: + continue + entry = self._parse_single_entry(entry_text, change_type) + if entry: + entries.append(entry) + else: + # No standard entries found, try as paragraph + entry_text = section_content.strip() + if entry_text and entry_text != "(No changes)": + entry = self._parse_single_entry(entry_text, change_type) + if entry: + entries.append(entry) + + return entries + + def _parse_single_entry(self, entry_text: str, change_type: str) -> Optional[ChangeEntry]: + """Parse a single entry into a ChangeEntry object.""" + # Extract authors + description, authors = AuthorParser.parse_authors(entry_text) + + # Extract issues/PRs + links = IssueExtractor.extract_issues(description) + + # Remove all issue/PR IDs from the description text + # Handle multiple formats of issue references at the beginning: + + # 1. Remove leading issues with mixed projects: "LUCENE-3323,SOLR-2659,LUCENE-3329,SOLR-2666: description" + description = re.sub(r'^(?:(?:SOLR|LUCENE|INFRA)-\d+(?:\s*[,:]?\s*)?)+:\s*', '', description) + + # 2. Remove SOLR-specific issues: "SOLR-12345: description" or "SOLR-12345, SOLR-12346: description" + description = re.sub(r'^(?:SOLR-\d+(?:\s*,\s*SOLR-\d+)*\s*[:,]?\s*)+', '', description) + + # 3. Remove PR references: "PR#123: description" or "GITHUB#456: description" + description = re.sub(r'^(?:(?:PR|GITHUB)#\d+(?:\s*,\s*(?:PR|GITHUB)#\d+)*\s*[:,]?\s*)+', '', description) + + # 4. Remove parenthesized issue lists at start: "(SOLR-123, SOLR-456)" + description = re.sub(r'^\s*\((?:SOLR-\d+(?:\s*,\s*)?)+\)\s*', '', description) + description = re.sub(r'^\s*\((?:(?:SOLR|LUCENE|INFRA)-\d+(?:\s*,\s*)?)+\)\s*', '', description) + + # 5. Remove any remaining leading issue references + description = re.sub(r'^[\s,;]*(?:SOLR-\d+|LUCENE-\d+|INFRA-\d+|PR#\d+|GITHUB#\d+)[\s,:;]*', '', description) + while re.match(r'^[\s,;]*(?:SOLR-\d+|LUCENE-\d+|INFRA-\d+|PR#\d+|GITHUB#\d+)', description): + description = re.sub(r'^[\s,;]*(?:SOLR-\d+|LUCENE-\d+|INFRA-\d+|PR#\d+|GITHUB#\d+)[\s,:;]*', '', description) + + description = description.strip() + + # Normalize whitespace: collapse multiple newlines/spaces into single spaces + # This joins multi-line formatted text into a single coherent paragraph + description = re.sub(r'\s+', ' ', description) + + # Escape HTML entities to prevent markdown rendering issues + # This converts <, >, &, etc. to <, >, & for safe markdown + description = html.escape(description) + + if not description: + return None + + return ChangeEntry( + title=description, + change_type=change_type, + authors=authors, + links=links, + ) + + +class YamlWriter: + """Writes ChangeEntry objects to YAML files.""" + + @staticmethod + def write_entry(entry: ChangeEntry, slug: str, output_dir: Path): + """Write a single entry to a YAML file.""" + # Ensure output directory exists + output_dir.mkdir(parents=True, exist_ok=True) + + filename = f"{slug}.yml" + filepath = output_dir / filename + + # Convert entry to dictionary and write as YAML + entry_dict = entry.to_dict() + + with open(filepath, 'w', encoding='utf-8') as f: + # Use custom YAML dumper for better formatting + yaml.dump( + entry_dict, + f, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + width=80 # Line width for better readability + ) + + return filepath + + +class ReleaseDate: + """Fetches and manages release dates from Apache projects JSON.""" + + @staticmethod + def fetch_release_dates() -> dict: + """ + Fetch release dates from Apache projects JSON. + + Returns: + Dictionary mapping version strings to YYYY-MM-DD dates + Example: {'9.9.0': '2025-07-24', '9.8.1': '2025-03-11', ...} + """ + import urllib.request + + version_dates = {} + url = "https://projects.apache.org/json/projects/solr.json" + + try: + response = urllib.request.urlopen(url, timeout=10) + data = json.loads(response.read().decode('utf-8')) + + releases = data.get('release', []) + for release in releases: + version = release.get('revision') + created = release.get('created') + + if version and created: + version_dates[version] = created + except Exception as e: + print(f"Warning: Could not fetch release dates: {e}", file=sys.stderr) + + return version_dates + + +class MigrationRunner: + """Orchestrates the complete migration process.""" + + def __init__(self, changes_file_path: str, output_base_dir: str): + self.changes_file_path = changes_file_path + self.output_base_dir = Path(output_base_dir) + self.parser = ChangesParser(changes_file_path) + self.version_dates = ReleaseDate.fetch_release_dates() + self.stats = { + 'versions_processed': 0, + 'entries_migrated': 0, + 'entries_skipped': 0, + 'files_created': 0, + 'release_dates_written': 0, + } + + def run(self): + """Execute the migration.""" + print(f"Parsing CHANGES.txt from: {self.changes_file_path}") + self.parser.parse() + + print(f"Found {len(self.parser.versions)} versions") + + for version_section in self.parser.versions: + self._process_version(version_section) + + self._print_summary() + + def _process_version(self, version_section: VersionSection): + """Process all entries for a single version.""" + version_dir = self.output_base_dir / version_section.get_directory_name() + + print(f"\nProcessing version {version_section.version}:") + print(f" Found {len(version_section.entries)} entries") + + # Write release-date.txt if we have a date for this version + if version_section.version in self.version_dates: + release_date = self.version_dates[version_section.version] + release_date_file = version_dir / "release-date.txt" + version_dir.mkdir(parents=True, exist_ok=True) + + with open(release_date_file, 'w', encoding='utf-8') as f: + f.write(release_date + '\n') + + self.stats['release_dates_written'] += 1 + print(f" Release date: {release_date}") + + entry_counter = 0 # For entries without explicit issue IDs + + for entry in version_section.entries: + # Find primary issue ID from links + issue_id = None + for link in entry.links: + if link.name.startswith('SOLR-'): + issue_id = link.name + break + + if not issue_id: + # If no SOLR issue found, try to use other JIRA/PR formats + for link in entry.links: + if link.name.startswith(('LUCENE-', 'INFRA-', 'PR#', 'GITHUB#')): + issue_id = link.name + break + + if not issue_id: + # No standard issue/PR found, generate a synthetic ID + # Use format: unknown-001, unknown-002, etc. + entry_counter += 1 + synthetic_id = f"unknown-{entry_counter:03d}" + issue_id = synthetic_id + + # Generate slug and write YAML + slug = SlugGenerator.generate_slug(issue_id, entry.title) + filepath = YamlWriter.write_entry(entry, slug, version_dir) + + print(f" ✓ {slug}.yml") + self.stats['entries_migrated'] += 1 + self.stats['files_created'] += 1 + + self.stats['versions_processed'] += 1 + + def _print_summary(self): + """Print migration summary.""" + print("\n" + "="*60) + print("Migration Summary:") + print(f" Versions processed: {self.stats['versions_processed']}") + print(f" Entries migrated: {self.stats['entries_migrated']}") + print(f" Entries skipped: {self.stats['entries_skipped']}") + print(f" Files created: {self.stats['files_created']}") + print(f" Release dates written: {self.stats['release_dates_written']}") + print("="*60) + + +def main(): + """Main entry point.""" + if len(sys.argv) < 2: + print("Usage: changes2logchange.py []") + print() + print("Arguments:") + print(" CHANGES.txt Path to the CHANGES.txt file to migrate") + print(" output_dir Directory to write changelog/ structure (default: ./changelog)") + sys.exit(1) + + changes_file = sys.argv[1] + output_dir = sys.argv[2] if len(sys.argv) > 2 else "changelog" + + if not os.path.exists(changes_file): + print(f"Error: CHANGES.txt file not found: {changes_file}", file=sys.stderr) + sys.exit(1) + + runner = MigrationRunner(changes_file, output_dir) + runner.run() + + +if __name__ == "__main__": + main() From 0e099abd9479d3b0a8b58a7f03a4db0dabb61485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 00:34:51 +0200 Subject: [PATCH 21/64] Updated docs to mention no need for changelog for minor issues --- dev-docs/changelog.adoc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 7ff3ebf32583..86fcd564a885 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -81,16 +81,15 @@ Invoke the task with: * Keep the entry short and focused on the user impact. * Choose the correct *type*: ** `added` for new features - ** `changed` for improvements - ** `optimized` for optimizations (performance, memory, etc.) + ** `changed` for improvements to existing code ** `fixed` for bug fixes ** `deprecated` for deprecated features ** `removed` for code removed in major releases ** `dependency_update` for updates to dependencies ** `security` for security-related changes ** `other` for anything else, like the build or documentation - ** `upgrade_notes` for upgrade notes, only used in older versions -* Reference issues as `SOLR-12345` or GitHub `#123`. +* Reference issues as `SOLR-12345` or GitHub `PR#123`. +* If the change is super minor, like a typo, don't bother adding a yaml file == 5. Example YAML Fragment From 9216ee841e1979b4261465349846979d4250efbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 00:35:16 +0200 Subject: [PATCH 22/64] Remove the optimized type --- changelog/logchange-config.yml | 23 ++++++++++------------- dev-tools/scripts/changes2logchange.py | 12 +++--------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 651f609f0b91..2a0455d7584a 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -25,30 +25,27 @@ changelog: order: 2 - key: changed order: 3 - - key: optimized - order: 4 - key: fixed - order: 5 + order: 4 - key: deprecated - order: 6 + order: 5 - key: removed - order: 7 + order: 6 - key: dependency_update - order: 8 + order: 7 - key: security - order: 9 + order: 8 - key: other - order: 10 + order: 9 labels: unreleased: unreleased important_notes: Important notes types: entryTypesLabels: - added: New Features - changed: Improvements - optimized: Optimizations - fixed: Bug Fixes - deprecated: Deprecations + added: Added + changed: Changed + fixed: Fixed + deprecated: Deprecated removed: Removed dependency_update: Dependency Upgrades security: Security diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py index 5d94c45731db..c76cf723fee8 100755 --- a/dev-tools/scripts/changes2logchange.py +++ b/dev-tools/scripts/changes2logchange.py @@ -62,9 +62,9 @@ class ChangeType: "Improvements / Changes": "changed", # Performance / Optimizations - "Optimizations": "optimized", - "Performance": "optimized", - "Optimization": "optimized", + "Optimizations": "changed", + "Performance": "changed", + "Optimization": "changed", # Bug Fixes "Bug Fixes": "fixed", @@ -101,12 +101,6 @@ class ChangeType: # Upgrade Notes - special category "Upgrade Notes": "upgrade_notes", - # Sections to skip (no migration) - # These are handled specially and should not create entries - # "Versions of Major Components": SKIP, - # "Detailed Change List": SKIP, - # etc. - # Other "Other Changes": "other", "Other": "other", From b995e6f8f4168307a4bd238023d6cd7a473de5f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 00:41:31 +0200 Subject: [PATCH 23/64] Doc change --- dev-docs/changelog.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 86fcd564a885..036da696c685 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -3,9 +3,9 @@ :toclevels: 2 :icons: font -Since Solr 10.0 we no longer edit a monolithic *CHANGES.txt* file directly. -Instead, we create a small YAML fragment **per bug-fix or feature**, checked into -`solr/changelog/unreleased/`. At release time these fragments are automatically +The project no longer maintains a monolithic *CHANGES.txt* file directly. +Instead, we create small YAML fragments **per bug-fix or feature**, checked into +`solr/changelog/v1.2.3/`. At release time these fragments are automatically collated into a human-readable *CHANGES.md*. This document walks you through the workflow, explains the Gradle helpers From be619cf6311e5729407d27b8744537fabc0b2570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 01:23:01 +0200 Subject: [PATCH 24/64] Generate newest entries in 'unreleased' folder --- dev-tools/scripts/README.md | 20 ++--- dev-tools/scripts/changes2logchange.py | 108 +++++++++++++++++++------ 2 files changed, 93 insertions(+), 35 deletions(-) diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.md index 0681b069d02a..438be543fa3b 100644 --- a/dev-tools/scripts/README.md +++ b/dev-tools/scripts/README.md @@ -183,20 +183,22 @@ files for each changelog entry, organized by version (v10.0.0/, v9.9.0/, etc.). Each YAML file complies with the schema outlined in `dev-docs/changelog.adoc`. - usage: changes2logchange.py [] + usage: changes2logchange.py [-h] [-o OUTPUT_DIR] [--last-released VERSION] changes_file Positional arguments: - CHANGES.txt Path to the CHANGES.txt file to migrate - output_dir Directory to write changelog/ structure (default: ./changelog) + changes_file Path to the CHANGES.txt file to migrate + + Optional arguments: + -h, --help Show this help message and exit + -o OUTPUT_DIR, --output-dir OUTPUT_DIR + Output directory for changelog structure (default: ./changelog) + --last-released VERSION Override auto-detected latest released version (e.g., 9.5.0) + Versions newer than this will be routed to unreleased/ folder Example usage: - python3 dev-tools/scripts/changes2logchange.py solr/CHANGES.txt changelog/ - Output: - - Creates vX.Y.Z/ directories for each version - - Generates SOLR-XXXXX-slug-name.yml files for entries with JIRA issues - - Skips entries without traceable issue/PR references - - Reports migration statistics (versions, entries migrated, entries skipped) + # Default behavior + python3 dev-tools/scripts/changes2logchange.py solr/CHANGES.txt ### gitignore-gen.sh diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py index c76cf723fee8..18f9d64e125b 100755 --- a/dev-tools/scripts/changes2logchange.py +++ b/dev-tools/scripts/changes2logchange.py @@ -590,17 +590,21 @@ class ReleaseDate: """Fetches and manages release dates from Apache projects JSON.""" @staticmethod - def fetch_release_dates() -> dict: + def fetch_release_dates_and_latest() -> tuple: """ - Fetch release dates from Apache projects JSON. + Fetch release dates from Apache projects JSON and identify latest version. Returns: - Dictionary mapping version strings to YYYY-MM-DD dates - Example: {'9.9.0': '2025-07-24', '9.8.1': '2025-03-11', ...} + Tuple of (version_dates_dict, latest_version_string) + Example: ({'9.9.0': '2025-07-24', ...}, '9.9.0') """ import urllib.request + from packaging import version as pkg_version version_dates = {} + latest_version = None + latest_version_obj = None + url = "https://projects.apache.org/json/projects/solr.json" try: @@ -609,31 +613,51 @@ def fetch_release_dates() -> dict: releases = data.get('release', []) for release in releases: - version = release.get('revision') + ver = release.get('revision') created = release.get('created') - if version and created: - version_dates[version] = created + if ver and created: + version_dates[ver] = created + + # Track the latest (highest) version + try: + ver_obj = pkg_version.parse(ver) + if latest_version_obj is None or ver_obj > latest_version_obj: + latest_version_obj = ver_obj + latest_version = ver + except Exception: + # Skip invalid version strings + pass except Exception as e: print(f"Warning: Could not fetch release dates: {e}", file=sys.stderr) - return version_dates + return version_dates, latest_version class MigrationRunner: """Orchestrates the complete migration process.""" - def __init__(self, changes_file_path: str, output_base_dir: str): + def __init__(self, changes_file_path: str, output_base_dir: str, last_released_version: Optional[str] = None): self.changes_file_path = changes_file_path self.output_base_dir = Path(output_base_dir) self.parser = ChangesParser(changes_file_path) - self.version_dates = ReleaseDate.fetch_release_dates() + + # Fetch release dates and latest version + self.version_dates, detected_latest = ReleaseDate.fetch_release_dates_and_latest() + + # Use provided version or detected latest + self.last_released_version = last_released_version or detected_latest + + if self.last_released_version: + print(f"Latest released version: {self.last_released_version}", file=sys.stderr) + self.stats = { 'versions_processed': 0, 'entries_migrated': 0, 'entries_skipped': 0, 'files_created': 0, 'release_dates_written': 0, + 'unreleased_entries': 0, } def run(self): @@ -650,9 +674,28 @@ def run(self): def _process_version(self, version_section: VersionSection): """Process all entries for a single version.""" - version_dir = self.output_base_dir / version_section.get_directory_name() + from packaging import version as pkg_version + + # Determine if this version should go to unreleased folder + is_unreleased = False + if self.last_released_version: + try: + current_ver = pkg_version.parse(version_section.version) + latest_ver = pkg_version.parse(self.last_released_version) + is_unreleased = current_ver > latest_ver + except Exception: + # If parsing fails, treat as unreleased (conservative approach) + is_unreleased = True + + # Route to appropriate directory + if is_unreleased: + version_dir = self.output_base_dir / "unreleased" + print(f"\nProcessing version {version_section.version} (unreleased):") + self.stats['unreleased_entries'] += len(version_section.entries) + else: + version_dir = self.output_base_dir / version_section.get_directory_name() + print(f"\nProcessing version {version_section.version}:") - print(f"\nProcessing version {version_section.version}:") print(f" Found {len(version_section.entries)} entries") # Write release-date.txt if we have a date for this version @@ -710,27 +753,40 @@ def _print_summary(self): print(f" Entries skipped: {self.stats['entries_skipped']}") print(f" Files created: {self.stats['files_created']}") print(f" Release dates written: {self.stats['release_dates_written']}") + if self.stats['unreleased_entries'] > 0: + print(f" Unreleased entries: {self.stats['unreleased_entries']}") print("="*60) def main(): """Main entry point.""" - if len(sys.argv) < 2: - print("Usage: changes2logchange.py []") - print() - print("Arguments:") - print(" CHANGES.txt Path to the CHANGES.txt file to migrate") - print(" output_dir Directory to write changelog/ structure (default: ./changelog)") - sys.exit(1) - - changes_file = sys.argv[1] - output_dir = sys.argv[2] if len(sys.argv) > 2 else "changelog" - - if not os.path.exists(changes_file): - print(f"Error: CHANGES.txt file not found: {changes_file}", file=sys.stderr) + import argparse + + parser = argparse.ArgumentParser( + description="Migrate Apache Solr CHANGES.txt to logchange YAML format" + ) + parser.add_argument( + "changes_file", + help="Path to the CHANGES.txt file to migrate" + ) + parser.add_argument( + "-o", "--output-dir", + default="changelog", + help="Directory to write changelog/ structure (default: ./changelog)" + ) + parser.add_argument( + "--last-released", + help="Last released version (e.g., 9.9.0). Versions newer than this go to unreleased/. " + "If not specified, fetches from Apache projects JSON." + ) + + args = parser.parse_args() + + if not os.path.exists(args.changes_file): + print(f"Error: CHANGES.txt file not found: {args.changes_file}", file=sys.stderr) sys.exit(1) - runner = MigrationRunner(changes_file, output_dir) + runner = MigrationRunner(args.changes_file, args.output_dir, args.last_released) runner.run() From 463892d54b0b69b4b7a06fb1392f7a6c3efe6d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 01:47:07 +0200 Subject: [PATCH 25/64] Add a github workflow to warn about lack of changelog --- .github/workflows/validate-changelog.yml | 88 ++++++++++++++++++++++++ dev-docs/changelog.adoc | 15 +++- gradle/changelog.gradle | 4 +- 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/validate-changelog.yml diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml new file mode 100644 index 000000000000..2ea4c41bb74e --- /dev/null +++ b/.github/workflows/validate-changelog.yml @@ -0,0 +1,88 @@ +name: Validate Changelog + +on: + pull_request: + branches: + - '*' + +jobs: + validate-changelog: + name: Check changelog entry + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Check for no-changelog label + id: check-label + run: | + LABELS='${{ toJson(github.event.pull_request.labels) }}' + if echo "$LABELS" | grep -q '"no-changelog"'; then + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + + - name: Check for CHANGES.txt edits + if: steps.check-label.outputs.skip == 'false' + run: | + # Get the list of changed files + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + if echo "$CHANGED_FILES" | grep -q "^solr/CHANGES\.txt$"; then + echo "::error::Use of solr/CHANGES.txt is disallowed. Please create a changelog yaml file instead." + echo "" + echo "Instead of editing CHANGES.txt, please:" + echo "1. Run: ./gradlew generateChangeYaml" + echo "2. Edit the generated YAML file in changelog/unreleased/" + echo "3. Commit both the code change and the YAML file" + echo "" + echo "For more information, see: dev-docs/changelog.adoc" + echo "" + echo "If this PR should not have a changelog entry (e.g., documentation-only changes)," + echo "add the 'no-changelog' label to this PR." + exit 1 + fi + + - name: Check for changelog entry + if: steps.check-label.outputs.skip == 'false' + run: | + # Get the list of changed files + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + # Check if any files were added to changelog/unreleased/ + if echo "$CHANGED_FILES" | grep -q "^solr/changelog/unreleased/"; then + echo "✓ Changelog entry found" + exit 0 + fi + + # Check if only docs/tests/comments were changed (common exceptions) + HAS_NON_DOCS_CHANGES=false + while IFS= read -r file; do + # Skip changelog, docs, tests, and certain config files + if ! echo "$file" | grep -qE "(^solr/changelog/|^solr/solr-ref-guide/|^dev-docs/|\.md$|\.adoc$|^solr/.*/test|\.gradle$|\.properties$|README|NOTICE|LICENSE)"; then + HAS_NON_DOCS_CHANGES=true + break + fi + done <<< "$CHANGED_FILES" + + if [ "$HAS_NON_DOCS_CHANGES" = false ]; then + echo "✓ No code changes detected (docs/tests only)" + exit 0 + fi + + echo "::error::This PR appears to contain code changes but no changelog entry was added." + echo "" + echo "Please add a changelog entry by:" + echo "1. Running: ./gradlew generateChangeYaml" + echo "2. Editing the generated YAML file in changelog/unreleased/" + echo "3. Committing the YAML file" + echo "" + echo "For more information, see: dev-docs/changelog.adoc" + echo "" + echo "If this PR should not have a changelog entry (e.g., refactoring, internal cleanup)," + echo "add the 'no-changelog' label to this PR." + exit 1 diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 036da696c685..50fc2f22f650 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -75,7 +75,16 @@ Invoke the task with: | Run by the Release Manager during the release procedure. |=== -== 4. Writing Good Entries +== 4. Changelog Validation in Pull Requests + +The `validate-changelog` GitHub workflow automatically checks that: + +. **CHANGES.txt is not edited directly** - All changes must use the YAML fragment approach +. **A changelog entry is added** - Code changes must include a corresponding YAML file in `changelog/unreleased/` + +If your PR modifies code but should not have a changelog entry (e.g., internal refactoring, build system changes), add the `no-changelog` label to skip these checks. + +== 5. Writing Good Entries * **Audience** is end-users and administrators, not committers. * Keep the entry short and focused on the user impact. @@ -91,7 +100,7 @@ Invoke the task with: * Reference issues as `SOLR-12345` or GitHub `PR#123`. * If the change is super minor, like a typo, don't bother adding a yaml file -== 5. Example YAML Fragment +== 6. Example YAML Fragment [source, yaml] ---- @@ -107,7 +116,7 @@ links: url: https://issues.apache.org/jira/browse/SOLR-3333 ---- -== 6. Further Reading +== 7. Further Reading * xref:https://github.com/logchange/logchange[Logchange web page] * xref:https://keepachangelog.com/en/1.1.0/[keepachangelog.com website] diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 9085a7679c4c..1f7d915193ff 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -39,13 +39,13 @@ task generateChangeYaml { def githubMatcher = gitBranch =~ /(PR|GH|GITHUB|#)?(\d+)/ def githubRef = githubMatcher ? githubMatcher.group(2) : "" def githubLink = githubMatcher ? "issues:\n - ${githubRef}" : "" - def branchWithoutJira = gitBranch.replaceFirst(/SOLR-\d+-/, "").replaceFirst(/(PR|GH|GITHUB|#)?(\d+)/, "").replace("-", " ").capitalize() + def branchWithoutJira = gitBranch.replaceFirst(/(SOLR|LUCENE|INFRA)-\d+-/, "").replaceFirst(/(PR|GH|GITHUB|#)?(\d+)/, "").replace("-", " ").capitalize() def fileName = "changelog/unreleased/${gitBranch}.yml" def file = new File(fileName) file.parentFile.mkdirs() file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format title: ${branchWithoutJira} -type: other # added, changed, optimized, fixed, deprecated, removed, dependency_update, security, other +type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other authors: - name: ${configuredName}${nick}${asfIdUrl} ${jiraLinks} From 31992b29a5decd56d42bacdb2aa4a1005fc5437e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 02:15:50 +0200 Subject: [PATCH 26/64] Update PR template --- .github/PULL_REQUEST_TEMPLATE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 60338d389cf4..4fd94fb8c383 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ Before creating a pull request, please file an issue in the ASF Jira system for * https://issues.apache.org/jira/projects/SOLR -For something minor (i.e. that wouldn't be worth putting in release notes), you can skip JIRA. +For something minor (i.e. that wouldn't be worth putting in release notes), you can skip JIRA. To create a Jira issue, you will need to create an account there first. The title of the PR should reference the Jira issue number in the form: @@ -42,3 +42,4 @@ Please review the following and check all that apply: - [ ] I have run `./gradlew check`. - [ ] I have added tests for my changes. - [ ] I have added documentation for the [Reference Guide](https://github.com/apache/solr/tree/main/solr/solr-ref-guide) +- [ ] I have added a [changelog entry](https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc) for my change From 7be559dcf82dd849d052ba8d8e4a520806841903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 02:18:29 +0200 Subject: [PATCH 27/64] Update addVersions.py --- dev-tools/scripts/addVersion.py | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/dev-tools/scripts/addVersion.py b/dev-tools/scripts/addVersion.py index 35a35cd35831..c7b752737d08 100755 --- a/dev-tools/scripts/addVersion.py +++ b/dev-tools/scripts/addVersion.py @@ -24,25 +24,6 @@ import re from configparser import ConfigParser, ExtendedInterpolation - -def update_changes(filename, new_version, init_changes, headers): - print(' adding new section to %s...' % filename, end='', flush=True) - matcher = re.compile(r'\d+\.\d+\.\d+\s+===') - def edit(buffer, match, line): - if new_version.dot in line: - return None - match = new_version.previous_dot_matcher.search(line) - if match is not None: - buffer.append(line.replace(match.group(0), new_version.dot)) - buffer.append(init_changes) - for header in headers: - buffer.append('%s\n---------------------\n(No changes)\n\n' % header) - buffer.append(line) - return match is not None - - changed = update_file(filename, matcher, edit) - print('done' if changed else 'uptodate') - def update_solrversion_class(new_version): filename = 'solr/api/src/java/org/apache/solr/client/api/util/SolrVersion.java' print(' changing version to %s...' % new_version.dot, end='', flush=True) @@ -54,7 +35,7 @@ def edit(buffer, match, line): return None buffer.append(line.replace(match.group(1), new_version.dot)) return True - + changed = update_file(filename, matcher, edit) print('done' if changed else 'uptodate') @@ -65,7 +46,7 @@ def edit(buffer, match, line): if new_version.dot in line: return None buffer.append(' String baseVersion = \'' + new_version.dot + '\'\n') - return True + return True changed = update_file(filename, version_prop_re, edit) print('done' if changed else 'uptodate') @@ -113,7 +94,7 @@ def check_lucene_match_version_tests(): print('ok') def read_config(current_version, current_lucene_version): - parser = argparse.ArgumentParser(description='Add a new version to CHANGES, to Version.java, build.gradle and solrconfig.xml files') + parser = argparse.ArgumentParser(description='Add a new version to Version.java, build.gradle and solrconfig.xml files') parser.add_argument('version', type=Version.parse, help='New Solr version') parser.add_argument('-l', dest='lucene_version', type=Version.parse, help='Optional lucene version. By default will read gradle/libs.versions.toml') newconf = parser.parse_args() @@ -137,18 +118,13 @@ def parse_properties_file(filename): def get_solr_init_changes(): return '' - + def main(): if not os.path.exists('build.gradle'): sys.exit("Tool must be run from the root of a source checkout.") current_version = Version.parse(find_current_version()) current_lucene_version = Version.parse(find_current_lucene_version()) newconf = read_config(current_version, current_lucene_version) - is_bugfix = newconf.version.is_bugfix_release() - - print('\nAdding new version %s' % newconf.version) - update_changes('solr/CHANGES.txt', newconf.version, get_solr_init_changes(), - ['Bug Fixes', 'Dependency Upgrades'] if is_bugfix else ['New Features', 'Improvements', 'Optimizations', 'Bug Fixes', 'Dependency Upgrades', 'Other Changes']) if newconf.is_latest_version: print('\nAdded version is latest version, updating...') From b10c09b1787e1c81caa9b1aa8ac38f18e56b391d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 02:29:13 +0200 Subject: [PATCH 28/64] Update addDepsToChanges.py to write yaml --- dev-tools/scripts/addDepsToChanges.py | 125 +++++++++++++------------- 1 file changed, 61 insertions(+), 64 deletions(-) diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py index 1fb5ceb7f5d4..1e1ab152f667 100755 --- a/dev-tools/scripts/addDepsToChanges.py +++ b/dev-tools/scripts/addDepsToChanges.py @@ -16,7 +16,7 @@ # limitations under the License. """ -Script to add solrbot changes lines to CHANGES.txt +Script to create changelog YAML entries for solrbot dependency updates """ import os import sys @@ -26,6 +26,8 @@ import argparse import re +import yaml +from pathlib import Path line_re = re.compile(r"(.*?) (\(branch_\d+x\) )?\(#(\d+)\)$") @@ -58,6 +60,44 @@ def __str__(self) -> str: # Keep trailing newline to preserve existing blank-line formatting by update_changes return f"* PR#{self.pr_num}: {self.message} ({self.author})\n" + def to_yaml_dict(self) -> dict: + """ + Convert to a dictionary suitable for YAML serialization + """ + return { + 'title': self.message, + 'type': 'dependency_update', + 'authors': [ + { + 'name': self.author + } + ], + 'links': [ + { + 'name': f'PR#{self.pr_num}', + 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' + } + ] + } + + def yaml_filename(self) -> str: + """ + Generate a filesystem-safe filename for this entry. + Format: PR#####-slug.yaml + """ + # Clean message for slug + slug = self.message.lower() + # Replace spaces with dashes + slug = re.sub(r'\s+', '-', slug) + # Remove non-alphanumeric except dashes + slug = re.sub(r'[^a-z0-9-]', '', slug) + # Truncate to reasonable length + slug = slug[:50] + # Remove trailing dashes + slug = slug.rstrip('-') + + return f"PR{self.pr_num}-{slug}.yaml" + def get_prev_release_tag(ver): """ @@ -119,72 +159,28 @@ def parse_gitlog_lines(lines, author: str): return entries -def gitlog_to_changes(line, user="solrbot"): +def write_changelog_yaml(entries): """ - DEPRECATED: Use parse_gitlog_lines + ChangeEntry.__str__ instead. - Converts a git log formatted line ending in (# Date: Tue, 21 Oct 2025 02:56:04 +0200 Subject: [PATCH 29/64] New changes2html.py script and change gradle task --- changelog/logchange-config.yml | 2 +- gradle/documentation/changes-to-html.gradle | 57 +- .../changes-to-html/changes2html.pl | 1049 ----------------- .../changes-to-html/changes2html.py | 624 ++++++++++ 4 files changed, 642 insertions(+), 1090 deletions(-) delete mode 100755 gradle/documentation/changes-to-html/changes2html.pl create mode 100755 gradle/documentation/changes-to-html/changes2html.py diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 2a0455d7584a..957a99ea01ed 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -17,7 +17,7 @@ # More info about configuration you can find https://github.com/logchange/logchange#configuration changelog: - heading: This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc + heading: This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. [https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc](https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc) entryTypes: # you can define custom entry types - key: upgrade_notes order: 1 diff --git a/gradle/documentation/changes-to-html.gradle b/gradle/documentation/changes-to-html.gradle index af9d1b5fa9a6..0f21cfca5061 100644 --- a/gradle/documentation/changes-to-html.gradle +++ b/gradle/documentation/changes-to-html.gradle @@ -20,7 +20,7 @@ def resources = scriptResources(buildscript) configure(project(':solr:documentation')) { task changesToHtml(type: ChangesToHtmlTask) { siteDir = resources - script = file("${resources}/changes2html.pl") + script = file("${resources}/changes2html.py") } // Make the rendered HTML of changes available as a separate @@ -36,7 +36,7 @@ configure(project(':solr:documentation')) { } } -// compile changes.txt into an html file +// compile CHANGELOG.md into an html file class ChangesToHtmlTask extends DefaultTask { @Internal @@ -46,10 +46,7 @@ class ChangesToHtmlTask extends DefaultTask { String productName = productProject.name @InputFile - File changesFile = productProject.file('CHANGES.txt') - - @InputFile - File changesDoapFile = project.rootProject.file("dev-tools/doap/${productName}.rdf") + File changesFile = project.rootProject.file('CHANGELOG.md') @InputDirectory File siteDir @@ -58,44 +55,27 @@ class ChangesToHtmlTask extends DefaultTask { final DirectoryProperty targetDir = project.objects.directoryProperty() .fileProvider(project.providers.provider { project.file("${project.docroot}/changes") }) - @Input - def luceneDocUrl = "${-> project.luceneDocUrl }" - @InputFile def script - def loadVersions(File outfile) { - // load version properties from DOAP RDF - def prefix = "doap.${productName}".toString() - ant.xmlproperty(keeproot: false, file: changesDoapFile, collapseAttributes: false, prefix: "${prefix}") - outfile.withWriter("UTF-8") { writer -> - writer.println(ant.properties["${prefix}.Project.release.Version.revision"]) - writer.println(ant.properties["${prefix}.Project.release.Version.created"]) - } - } - - def toHtml(File versionsFile) { + def toHtml() { def output = new ByteArrayOutputStream() - - // Check if the perl executable exists - if (!perlExists()) { - logger.warn("WARNING: Perl is not installed, skipping creating Changes.html") + + // Check if Python is available + if (!pythonExists()) { + logger.warn("WARNING: Python is not installed, skipping creating Changes.html") return } def result = project.exec { - executable project.externalTool("perl") - standardInput changesFile.newInputStream() + executable "python3" standardOutput project.file("${targetDir.get().getAsFile()}/Changes.html").newOutputStream() errorOutput = output ignoreExitValue = true args += [ - "-CSD", - script, - "${productName}", - versionsFile.toString(), - luceneDocUrl.concat('/') // slash required at end by perl script + script.toString(), + changesFile.toString() ] } @@ -107,24 +87,21 @@ class ChangesToHtmlTask extends DefaultTask { @TaskAction def convert() { project.mkdir targetDir - if (changesFile.exists() && changesDoapFile.exists()) { - File versionsFile = project.file("${project.buildDir}/doap.${project.name}.changes.version.dates.csv") - loadVersions(versionsFile) - toHtml(versionsFile) + if (changesFile.exists()) { + toHtml() project.copy { from siteDir into targetDir include "*.css" } - versionsFile.delete() } else { - throw new GradleException("Changes file ${changesFile} or Doap file ${changesDoapFile} not found.") + throw new GradleException("Changes file ${changesFile} not found.") } } - - def perlExists() { + + def pythonExists() { try { - def process = "perl -v".execute() + def process = "python3 --version".execute() process.waitFor() return process.exitValue() == 0 } catch (Exception e) { diff --git a/gradle/documentation/changes-to-html/changes2html.pl b/gradle/documentation/changes-to-html/changes2html.pl deleted file mode 100755 index 86f316467c8b..000000000000 --- a/gradle/documentation/changes-to-html/changes2html.pl +++ /dev/null @@ -1,1049 +0,0 @@ -#!/usr/bin/perl -# -# Transforms Lucene Core's or Solr's CHANGES.txt into Changes.html -# -# Input is on STDIN, output is to STDOUT -# -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -use strict; -use warnings; - -my $jira_url_prefix = 'http://issues.apache.org/jira/browse/'; -my $github_pull_request_prefix = 'https://github.com/apache/solr/pull/'; -my $month_regex = &setup_month_regex; -my %month_nums = &setup_month_nums; -my %lucene_bugzilla_jira_map = &setup_lucene_bugzilla_jira_map; -my $title = undef; -my $release = undef; -my $reldate = undef; -my $relinfo = undef; -my $sections = undef; -my $items = undef; -my $first_relid = undef; -my $second_relid = undef; -my @releases = (); - -my @lines = ; # Get all input at once - -# -# Cmdline args: (only from Solr) -# -my $product = uc($ARGV[0]); -if ($product !~ /^(LUCENE|SOLR)$/) { - print STDERR "Unknown product name '$ARGV[0]'\n"; - exit(1); -} -my %release_dates = &setup_release_dates($ARGV[1]); -my $lucene_javadoc_url = ($product eq 'SOLR' ? $ARGV[2] : ''); # Only Solr supplies this on the cmdline -my $in_major_component_versions_section = 0; - - -# -# Parse input and build hierarchical release structure in @releases -# -for (my $line_num = 0 ; $line_num <= $#lines ; ++$line_num) { - $_ = $lines[$line_num]; - unless (/\S/) { # Skip blank lines - $in_major_component_versions_section = 0; - next; - } - next if (/^\s{0,4}-{5,}\s*$/); # Skip Solr's section underlines - - unless ($title) { - if (/\S/) { - s/^[^\p{N}\p{L}]*//; # Trim leading non-alphanum chars, including BOM chars, if any - s/\s+$//; # Trim trailing whitespace - } - $title = $_; - next; - } - - if (/\s*===+\s*(.*?)\s*===+\s*/) { # New-style release headings - $release = $1; - $release =~ s/^(?:release|lucene)\s*//i; # Trim "Release " or "Lucene " prefix - ($release, $relinfo) = ($release =~ /^(\d+(?:\.(?:\d+))*(?:-(?:ALPHA|BETA))?|Trunk)\s*(.*)/i); - $relinfo =~ s/\s*:\s*$//; # Trim trailing colon - $relinfo =~ s/^\s*,\s*//; # Trim leading comma - ($reldate, $relinfo) = get_release_date($release, $relinfo); - $sections = []; - push @releases, [ $release, $reldate, $relinfo, $sections ]; - ($first_relid = 'v'.lc($release)) =~ s/\s+/_/g - if ($#releases == 0 or ($#releases == 1 and not ($releases[0][0]))); - ($second_relid = 'v'.lc($release)) =~ s/\s+/_/g - if ( ($#releases == 1 and $releases[0][0]) - or ($#releases == 2 and not $releases[0][0])); - $items = undef; - next; - } - - if (/^\s*([01](?:\.[0-9]{1,2}){1,2}[a-z]?(?:\s*(?:RC\d+|final))?)\s* - ((?:200[0-7]-.*|.*,.*200[0-7].*)?)$/x) { # Old-style release heading - $release = $1; - $relinfo = $2; - $relinfo =~ s/\s*:\s*$//; # Trim trailing colon - $relinfo =~ s/^\s*,\s*//; # Trim leading comma - ($reldate, $relinfo) = get_release_date($release, $relinfo); - $sections = []; - push @releases, [ $release, $reldate, $relinfo, $sections ]; - $items = undef; - next; - } - - if (m!^20\d\d[-/]\d{1,2}[-/]\d{1,2}!) { # Collect dated postscripts - my $item = $_; - my $line = ''; - while ($line_num < $#lines and ($line = $lines[++$line_num]) =~ /\S/) { - $line =~ s/^\s+//; # Trim leading whitespace - $line =~ s/\s+$//; # Trim trailing whitespace - $item .= "$line\n"; - } - push @releases, [ $item, '', '', [] ]; - next; - } - - # Section heading: no leading whitespace, initial word capitalized, - # six words or less, and no trailing punctuation, - # except colons - don't match the one otherwise matching - # non-section-name by excluding "StandardTokenizer" - if ( /^(?!.*StandardTokenizer)([A-Z]\S*(?:\s+\S+){0,5})(?[0]) { - die "Section '$heading' appears more than once under release '$releases[-1][0]'"; - } - } - push @$sections, [ $heading, $items ]; - $in_major_component_versions_section - = ($heading =~ /Versions of Major Components/i); - next; - } - - # Handle earlier releases without sections - create a headless section - unless ($items) { - $items = []; - unless (@releases) { - $sections = []; - # Make a fake release to hold pre-release sections and items - push @releases, [ undef, undef, undef, $sections ]; - } - push @$sections, [ '', $items ]; - } - - my $type; - if (@$items) { # A list item has been encountered in this section before - $type = $items->[0]; # 0th position of items array is list type - } else { - $type = get_list_type($_); - push @$items, $type; - } - - if ($type eq 'numbered') { # The modern items list style - # List item boundary is another numbered item or an unindented line - my $line; - my $item = $_; - $item =~ s/^(\s{0,2}\d+\.\d?\s*)//; # Trim the leading item number - my $leading_ws_width = length($1); - $item =~ s/\s+$//; # Trim trailing whitespace - $item .= "\n"; - - while ($line_num < $#lines - and ($line = $lines[++$line_num]) !~ /^(?:\s{0,2}\d+\.\s*\S|\S)/) { - $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace - $line =~ s/\s+$//; # Trim trailing whitespace - $item .= "$line\n"; - } - $item =~ s/\n+\Z/\n/; # Trim trailing blank lines - push @$items, $item; - --$line_num unless ($line_num == $#lines && $lines[$line_num] !~ /^20/); - } elsif ($type eq 'paragraph') { # List item boundary is a blank line - my $line; - my $item = $_; - $item =~ s/^(\s+)//; - my $leading_ws_width = defined($1) ? length($1) : 0; - $item =~ s/\s+$//; # Trim trailing whitespace - $item .= "\n"; - - unless ($in_major_component_versions_section) { - while ($line_num < $#lines and ($line = $lines[++$line_num]) =~ /\S/) { - $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace - $line =~ s/\s+$//; # Trim trailing whitespace - $item .= "$line\n"; - } - } else { - ++$line_num; - } - push @$items, $item; - --$line_num unless ($line_num == $#lines && $lines[$line_num] !~ /^20/); - } else { # $type is one of the bulleted types - # List item boundary is another bullet or a blank line - my $line; - my $item = $_; - $item =~ s/^(\s*\Q$type\E\s*)//; # Trim the leading bullet - my $leading_ws_width = defined($1) ? length($1) : 0; - $item =~ s/\s+$//; # Trim trailing whitespace - $item .= "\n"; - - while ($line_num < $#lines - and ($line = $lines[++$line_num]) !~ /^(?:\S|\s*\Q$type\E\s+)/) { - $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace - $line =~ s/\s+$//; # Trim trailing whitespace - $item .= "$line\n"; - } - push @$items, $item; - --$line_num unless ($line_num == $#lines && $lines[$line_num] !~ /^20/); - } -} - -# Recognize IDs of top level nodes of the most recent two releases, -# escaping JavaScript regex metacharacters, e.g.: "^(?:trunk|2\\\\.4\\\\.0)" -my $first_relid_regex = $first_relid; -$first_relid_regex =~ s!([.+*?{}()|^$/\[\]\\])!\\\\\\\\$1!g; -my $second_relid_regex = $second_relid; -$second_relid_regex =~ s!([.+*?{}()|^$/\[\]\\])!\\\\\\\\$1!g; -my $newer_version_regex = "^(?:$first_relid_regex|$second_relid_regex)"; - -# -# Print HTML-ified version to STDOUT -# -print<<"__HTML_HEADER__"; - - - - - $title - - - - - - - - -

$title

- -
- -__HTML_HEADER__ - -my $heading; -my $relcnt = 0; -my $header = 'h2'; -my $subheader = 'h3'; - -for my $rel (@releases) { - if ($relcnt == 2) { - $header = 'h3'; - $subheader = 'h4'; - print "

"; - print "Older Releases"; - print "

\n"; - print "
\n" - } - - ($release, $reldate, $relinfo, $sections) = @$rel; - - # The first section heading is undefined for the older sectionless releases - my $has_release_sections = has_release_sections($sections); - - my $relid = ''; - if ($release) { # Pre-release sections have no release ID - ++$relcnt; - ($relid = 'v'.lc($release)) =~ s/\s+/_/g; - print "<$header>"; - print "" - unless ($release =~ /^20\d\d/); - print "Release " unless ($release =~ /^trunk$|^20\d\d/i); - print "$release $relinfo"; - print " [$reldate]" unless ($reldate eq 'unknown' or not $reldate); - print "" unless ($release =~ /^20\d\d/); - print "\n"; - print "
    \n" - if ($has_release_sections); - } - - my $licnt = 0; - for my $section (@$sections) { - ($heading, $items) = @$section; - (my $sectid = lc($heading)) =~ s/\s+/_/g; - my $numItemsStr = $#{$items} > 0 ? "($#{$items})" : "(none)"; - - my $list_item = "li"; - if ($release) { - if ($heading) { - if ($heading eq 'Detailed Change List') { - print "
  • <$subheader>$heading
    • \n"; - next; - } elsif ($has_release_sections) { - print "
  • $heading", - "   $numItemsStr\n"; - ++$licnt; - } - } - } else { # $release is not defined - print "

    $heading

    \n" if ($heading); - $list_item = "p"; - } - - my $list_type = $items->[0] || ''; - my $list = ($has_release_sections || $list_type eq 'numbered' ? 'ol' : 'ul'); - my $listid = $sectid ? "$relid.$sectid" : $relid; - print " <$list id=\"$listid.list\">\n" - unless (not $release or ($has_release_sections and not $heading)); - - for my $itemnum (1..$#{$items}) { - my $item = $items->[$itemnum]; - $item =~ s:&:&:g; # Escape HTML metachars, but leave tags - $item =~ s~<(?!/?code>(?:[^,]|$))~<~gi; # intact - unless followed by a comma - and - $item =~ s:(?:>:gi; # add 
     wrappers for non-inline sections
-      $item =~ s{((?:^|.*\n)\s*)(?!,)(?!.+)(.+)(?![ \t]*\S)}
-                { 
-                  my $prefix = $1; 
-                  my $code = $2;
-                  $code =~ s/\s+$//;
-                  "$prefix
    $code>
    "
-                }gise;
-
-      $item = markup_trailing_attribution($item) unless ($item =~ /\n[ ]*-/);
-
-      $item =~ s{(.*?)(
    .*?
    )|(.*)}
-                {
-                  my $uncode = undef;
-                  my ($one,$two,$three) = ($1,$2,$3);
-                  if (defined($two)) {
-                    $uncode = $one || '';
-                    $uncode =~ s{^(.*?)(?=\n[ ]*-)}
-                                {
-                                  my $prefix = $1;
-                                  my ($primary,$additional_work) = $prefix =~ /^(.*?)((?:\s*Additional\s+Work:\s*)?)$/si;
-                                  my $result = markup_trailing_attribution($primary);
-                                  $result .= "
    \n$additional_work
    " if ($additional_work);
-                                  $result;
-                                }se;
-                    $uncode =~ s{((?<=\n)[ ]*-.*\n(?:.*\n)*)}
-                                {
-                                  my $bulleted_list = $1;
-                                  $bulleted_list
-                                    =~ s{(?:(?<=\n)|\A)[ ]*-[ ]*(.*(?:\n|\z)(?:[ ]+[^ -].*(?:\n|\z))*)}
-                                        {
-                                            qq!
  • \n!
-                                          . markup_trailing_attribution($1)
-                                          . "
    • \n"
-                                        }ge;
-                                  $bulleted_list
-                                    =~ s{(\n)(.*)}
-                                        {
-                                            qq!
      \n$1
    \n!
-                                          . markup_trailing_attribution($2 || '')
-                                        }se;
-                                  $bulleted_list;
-                                }ge;
-                    "$uncode$two";
-                  } else {
-                    $uncode = $three || '';
-                    $uncode =~ s{^(.*?)(?=\n[ ]*-)}
-                                {
-                                  my $prefix = $1;
-                                  my ($primary,$additional_work) = $prefix =~ /^(.*?)((?:\s*Additional\s+Work:\s*)?)$/si;
-                                  my $result = markup_trailing_attribution($primary);
-                                  $result .= "
    \n$additional_work
    " if ($additional_work);
-                                  $result;
-                                }se;
-                    $uncode =~ s{((?<=\n)[ ]*-.*\n(?:.*\n)*)}
-                                {
-                                  my $bulleted_list = $1;
-                                  $bulleted_list
-                                    =~ s{(?:(?<=\n)|\A)[ ]*-[ ]*(.*(?:\n|\z)(?:[ ]+[^ -].*(?:\n|\z))*)}
-                                        {
-                                            qq!
  • \n!
-                                          . markup_trailing_attribution($1)
-                                          . "
    • \n"
-                                        }ge;
-                                  $bulleted_list
-                                    =~ s{(\n)(.*)}
-                                        {
-                                            qq!
      \n$1
    \n!
-                                          . markup_trailing_attribution($2 || '')
-                                        }se;
-                                  $bulleted_list;
-                                }ge;
-                    $uncode = markup_trailing_attribution($uncode);
-                    $uncode;
-                  }
-                }sge;
-
-      $item =~ s:\n{2,}:\n
    \n:g;                   # Keep paragraph breaks
-      $item =~ s:
    • \n

    \n\n\n

    \n(.*)\n:

\n$1\n

    :g; - # Link LUCENE-XXX, SOLR-XXX and INFRA-XXX to JIRA - $item =~ s{(?:${jira_url_prefix})?((?:LUCENE|SOLR|INFRA)-\d+)} - {$1}g; - $item =~ s{(issue\s*\#?\s*(\d{3,}))} # Link Issue XXX to JIRA - {$1}gi; - # Link Lucene XXX, SOLR XXX and INFRA XXX to JIRA - $item =~ s{((LUCENE|SOLR|INFRA)\s+(\d{3,}))} - {$1}gi; - # Link "[ github | gh ] pull request [ # ] X+" to Github pull request - $item =~ s{((?:(?:(?:github|gh)\s+)?pull\s+request\s*(?:\#?\s*)?|gh-|(gh|github|pr)#)(\d+))} - {$1}gi; - # Link "LUCENE_CHANGES.txt" to Lucene's same-release Changes.html - # TODO: We can no longer rely on this since Solr may have a different Lucene version - # But it will still work for pre-9.0 changes, and from 9.0 we can instead provide - # full links. - if ($product eq 'SOLR') { - $item =~ s[(LUCENE_CHANGES.txt)] - [$1]g; - } - if ($product eq 'LUCENE') { - # Find single Bugzilla issues - $item =~ s~((?i:bug|patch|issue)\s*\#?\s*(\d+)) - ~ my $issue = $1; - my $jira_issue_num = $lucene_bugzilla_jira_map{$2}; # Link to JIRA copies - $issue = qq!! - . qq!$issue [LUCENE-$jira_issue_num]! - if (defined($jira_issue_num)); - $issue; - ~gex; - # Find multiple Bugzilla issues - $item =~ s~(?<=(?i:bugs))(\s*)(\d+)(\s*(?i:\&|and)\s*)(\d+) - ~ my $leading_whitespace = $1; - my $issue_num_1 = $2; - my $interlude = $3; - my $issue_num_2 = $4; - # Link to JIRA copies - my $jira_issue_1 = $lucene_bugzilla_jira_map{$issue_num_1}; - my $issue1 - = qq!! - . qq!$issue_num_1 [LUCENE-$jira_issue_1]! - if (defined($jira_issue_1)); - my $jira_issue_2 = $lucene_bugzilla_jira_map{$issue_num_2}; - my $issue2 - = qq!! - . qq!$issue_num_2 [LUCENE-$jira_issue_2]! - if (defined($jira_issue_2)); - $leading_whitespace . $issue1 . $interlude . $issue2; - ~gex; - } - - # Linkify URLs, except Bugzilla links, which don't work anymore - # also ignore localhost, ..., $SOLR_PORT, 127.0.0.1 - $item =~ s~(?])(https?://(?!(?:nagoya|issues)\.apache\.org/bugzilla|localhost|\.\.\.|\$SOLR_HOST|127\.0\.0\.1)[^\s\)]+)~$1~g; - - $item =~ s~
\s+

\s+~~; - - print " <$list_item>$item\n"; - } - print " \n" unless (not $release or ($has_release_sections and not $heading)); - if ($release and $has_release_sections and $licnt>0) { - print " \n"; - --$licnt; - } - } - print "\n" if ($release and $has_release_sections); -} -print "

\n" if ($relcnt > 3); -print "\n\n"; - - -# Subroutine: markup_trailing_attribution -# -# Takes one parameter: -# -# - text possibly containing a trailing parenthesized attribution -# -# Returns one scalar: -# -# - text with the trailing attribution, if any, marked up with the color green -# -sub markup_trailing_attribution { - my $item = shift; - - # Put attributions on their own lines - this already happens if there is a preceding - my $extra_newline = ($item =~ m::) ? '' : '
'; - # Check for trailing parenthesized attribution with no following period. - # Exclude things like "(see #3 above)" and "(use the bug number instead of xxxx)" - unless ($item =~ s{\s+(\((?![Ss]ee ) - (?!spans\b) - (?!mainly\ ) - (?!LUCENE-\d+\)) - (?!SOLR-\d+\)) - (?!user's) - (?!like\ ) - (?!r\d{6}) # subversion revision - (?!and\ ) - (?!backported\ ) - (?!in\ ) - (?!inverse\ ) - (?![Tt]he\ ) - (?!use\ the\ bug\ number) - (?!e\.?g\.?\b) - [^()"]+?\))\s*$} - {\n${extra_newline}$1}x) { - # If attribution is not found, then look for attribution with a - # trailing period, but try not to include trailing parenthesized things - # that are not attributions. - # - # Rule of thumb: if a trailing parenthesized expression with a following - # period does not contain "LUCENE-XXX", and it either has three or - # fewer words or it includes the word "via" or the phrase "updates from", - # then it is considered to be an attribution. - - $item =~ s{(\s+(\((?![Ss]ee\ ) - (?!spans\b) - (?!mainly\ ) - (?!LUCENE-\d+\)) - (?!SOLR-\d+\)) - (?!user's) - (?!like\ ) - (?!r\d{6}) # subversion revision - (?!and\ ) - (?!backported\ ) - (?!in\ ) - (?!inverse\ ) - (?![Tt]he\ ) - (?!use\ the\ bug\ number) - (?!e\.?g\.?\b) - [^()"]+?\))) - ((?:\.|(?i:\.?\s*Issue\s+\d{3,}|LUCENE-\d+)\.?)\s*)$} - { - my $subst = $1; # default: no change - my $parenthetical = $2; - my $trailing_period_and_or_issue = $3; - if ($parenthetical !~ /LUCENE-\d+/) { - my ($no_parens) = $parenthetical =~ /^\((.*)\)$/s; - my @words = grep {/\S/} split /\s+/, $no_parens; - my $commas = $no_parens =~ s/,/,/g; # count commas - my $max_words = 4 + $commas; - if ($no_parens =~ /\b(?:via|updates\s+from)\b/i || scalar(@words) <= $max_words) { - $subst = "\n${extra_newline}$parenthetical"; - } - } - $subst . $trailing_period_and_or_issue; - }ex; - } - return $item; -} - -# -# Subroutine: has_release_sections -# -# Takes one parameter: -# -# - The $sections array reference -# -# Returns one scalar: -# -# - A boolean indicating whether there are release sections -# -sub has_release_sections { - my $sections = shift; - my $has_release_sections = 0; - for my $section_num (0 .. $#{$sections}) { - if ($sections->[$section_num][0]) { - $has_release_sections = 1; - last; - } - } - return $has_release_sections; -} - - -# -# Subroutine: get_list_type -# -# Takes one parameter: -# -# - The first line of a sub-section/point -# -# Returns one scalar: -# -# - The list type: 'numbered'; or one of the bulleted types '-', or '.' or -# 'paragraph'. -# -sub get_list_type { - my $first_list_item_line = shift; - my $type = 'paragraph'; # Default to paragraph type - - if ($first_list_item_line =~ /^\s{0,2}\d+\.\s+\S+/) { - $type = 'numbered'; - } elsif ($first_list_item_line =~ /^\s*([-.*])\s+\S+/) { - $type = $1; - } - return $type; -} - - -# -# Subroutine: get_release_date -# -# Takes two parameters: -# -# - Release name -# - Release info, potentially including a release date -# -# Returns two scalars: -# -# - The release date, in format YYYY-MM-DD -# - The remainder of the release info (if any), with release date stripped -# -sub get_release_date { - my $release = shift; - my $relinfo = shift; - - my ($year, $month, $dom, $reldate); - - if ($relinfo) { - if ($relinfo =~ s:\s*(2\d\d\d)([-./]) - (1[012]|0?[1-9])\2 - ([12][0-9]|30|31|0?[1-9])\s*: :x) { - # YYYY-MM-DD or YYYY-M-D or YYYY-MM-D or YYYY-M-DD - $year = $1; - $month = $3; - $dom = $4; - $dom = "0$dom" if (length($dom) == 1); - $reldate = "$year-$month-$dom"; - } elsif ($relinfo =~ s:\s*(1[012]|0?[1-9])([-./]) - ([12][0-9]|30|31|0?[1-9])\2 - (2\d\d\d)\s*: :x) { - # MM-DD-YYYY or M-D-YYYY or MM-D-YYYY or M-DD-YYYY - $month = $1; - $dom = $3; - $dom = "0$dom" if (length($dom) == 1); - $year = $4; - $reldate = "$year-$month-$dom"; - } elsif ($relinfo =~ s:($month_regex)\s* - ([12][0-9]|30|31|0?[1-9])((st|rd|th)\.?)?,?\s* - (2\d\d\d)\s*: :x) { - # MMMMM DD, YYYY or MMMMM DDth, YYYY - $month = $month_nums{$1}; - $dom = $2; - $dom = "0$dom" if (length($dom) == 1); - $year = $5; - $reldate = "$year-$month-$dom"; - } elsif ($relinfo =~ s:([12][0-9]|30|31|0?[1-9])(\s+|[-/.]) - ($month_regex)\2 - (2\d\d\d)\s*: :x) { - # DD MMMMM YYYY - $dom = $1; - $dom = "0$dom" if (length($dom) == 1); - $month = $month_nums{$3}; - $year = $4; - $reldate = "$year-$month-$dom"; - } - } - - unless ($reldate) { # No date found in $relinfo - # Handle '1.2 RC6', which should be '1.2 final' - $release = '1.2 final' if ($release eq '1.2 RC6'); - - $reldate = ( exists($release_dates{$release}) - ? $release_dates{$release} - : 'unknown'); - } - - $relinfo =~ s/,?\s*$//; # Trim trailing comma and whitespace - - return ($reldate, $relinfo); -} - - -# -# setup_release_dates -# -# Returns a list of alternating release names and dates, for use in populating -# the %release_dates hash. -# -# Pulls release dates from the project DOAP file. -# -sub setup_release_dates { - my %release_dates = (); - my $file = shift; - open(FILE, "<$file") || die "could not open $file: $!"; - my $version_list = ; - my $created_list = ; - close(FILE); - - $version_list =~ s/^\s+|\s+$//g; - my @versions = split /\s*,\s*/, $version_list; - $created_list =~ s/^\s+|\s+$//g; - my @created = split /\s*,\s*/, $created_list; - - if (scalar(@versions) != scalar(@created)) { - die $file . " contains" . scalar(@versions) . " versions but " . scalar(@created) . " creation dates."; - } - my $date; - for my $pos (0..$#versions) { - $date = normalize_date($created[$pos]); - $release_dates{$versions[$pos]} = $date; - if ($versions[$pos] =~ /^([1-9]\d*\.\d+)([^.0-9].*|$)/) { - my $padded_version_name = "$1.0$2"; # Alias w/trailing ".0" - $release_dates{$padded_version_name} = $date; - } elsif ($versions[$pos] =~ /\.0(?=[^.0-9]|$)/) { - my $trimmed_version_name = $versions[$pos]; - $trimmed_version_name =~ s/\.0(?=[^.0-9]|$)//; # Alias w/o trailing ".0" - $release_dates{$trimmed_version_name} = $date; - } - } - return %release_dates; -} - -# -# normalize_date -# -# Left-zero-pads month and day-of-month to 2 digits in dates of format YYYY-(M)M-(D)D -# -sub normalize_date { - my $date = shift; - my ($year, $month, $dom) = $date =~ /^(2\d\d\d)-(\d+)-(\d+)$/; - return sprintf("%04d-%02d-%02d", $year, $month, $dom); -} - - -# -# setup_month_regex -# -# Returns a string containing a regular expression with alternations for -# the standard month representations in English. -# -sub setup_month_regex { - return '(?i:Jan(?:|\.|uary)|Feb(?:|\.|ruary)|Mar(?:|\.|ch)' - . '|Apr(?:|\.|il)|May|Jun(?:|\.|e)|Jul(?:|\.|y)|Aug(?:|\.|ust)' - . '|Sep(?:|\.|t(?:|\.|ember))|Oct(?:|\.|ober)|Nov(?:|\.|ember)' - . '|Dec(?:|\.|ember))'; -} - - -# -# setup_month_nums -# -# Returns a list of alternating English month representations and the two-digit -# month number corresponding to them, for use in populating the %month_nums -# hash. -# -sub setup_month_nums { - return ( 'Jan' => '01', 'Jan.' => '01', 'January' => '01', - 'Feb' => '02', 'Feb.' => '02', 'February' => '02', - 'Mar' => '03', 'Mar.' => '03', 'March' => '03', - 'Apr' => '04', 'Apr.' => '04', 'April' => '04', - 'May' => '05', - 'Jun' => '06', 'Jun.' => '06', 'June' => '06', - 'Jul' => '07', 'Jul.' => '07', 'July' => '07', - 'Aug' => '08', 'Aug.' => '08', 'August' => '08', - 'Sep' => '09', 'Sep.' => '09', - 'Sept' => '09', 'Sept.' => '09', 'September' => '09', - 'Oct' => '10', 'Oct.' => '10', 'October' => '10', - 'Nov' => '11', 'Nov.' => '11', 'November' => '11', - 'Dec' => '12', 'Dec.' => '12', 'December' => '12' ); -} - - -# -# setup_lucene_bugzilla_jira_map -# -# Returns a list of alternating Bugzilla bug IDs and LUCENE-* JIRA issue -# numbers, for use in populating the %lucene_bugzilla_jira_map hash -# -sub setup_lucene_bugzilla_jira_map { - return ( 4049 => 1, 4102 => 2, 4105 => 3, 4254 => 4, - 4555 => 5, 4568 => 6, 4754 => 7, 5313 => 8, - 5456 => 9, 6078 => 10, 6091 => 11, 6140 => 12, - 6292 => 13, 6315 => 14, 6469 => 15, 6914 => 16, - 6968 => 17, 7017 => 18, 7019 => 19, 7088 => 20, - 7089 => 21, 7275 => 22, 7412 => 23, 7461 => 24, - 7574 => 25, 7710 => 26, 7750 => 27, 7782 => 28, - 7783 => 29, 7912 => 30, 7974 => 31, 8307 => 32, - 8525 => 33, 9015 => 34, 9110 => 35, 9347 => 36, - 9454 => 37, 9782 => 38, 9853 => 39, 9906 => 40, - 9970 => 41, 10340 => 42, 10341 => 43, 10342 => 44, - 10343 => 45, 10849 => 46, 11109 => 47, 11359 => 48, - 11636 => 49, 11918 => 50, 12137 => 51, 12273 => 52, - 12444 => 53, 12569 => 54, 12588 => 55, 12619 => 56, - 12667 => 57, 12723 => 58, 12749 => 59, 12761 => 60, - 12950 => 61, 13102 => 62, 13166 => 63, 14028 => 64, - 14355 => 65, 14373 => 66, 14412 => 67, 14485 => 68, - 14585 => 69, 14665 => 70, 14900 => 71, 15739 => 72, - 16025 => 73, 16043 => 74, 16167 => 75, 16245 => 76, - 16364 => 77, 16437 => 78, 16438 => 79, 16470 => 80, - 16677 => 81, 16719 => 82, 16730 => 83, 16816 => 84, - 16952 => 85, 17242 => 86, 17954 => 88, 18014 => 89, - 18088 => 90, 18177 => 91, 18410 => 87, 18833 => 92, - 18847 => 93, 18914 => 94, 18927 => 95, 18928 => 96, - 18929 => 97, 18931 => 98, 18932 => 99, 18933 => 100, - 18934 => 101, 19058 => 102, 19149 => 103, 19189 => 104, - 19253 => 105, 19468 => 106, 19686 => 107, 19736 => 108, - 19751 => 109, 19834 => 110, 19844 => 111, 20024 => 112, - 20081 => 113, 20123 => 114, 20196 => 115, 20283 => 116, - 20290 => 117, 20461 => 118, 20901 => 119, 21128 => 120, - 21149 => 121, 21150 => 122, 21189 => 123, 21446 => 124, - 21921 => 126, 22344 => 128, 22469 => 130, 22987 => 131, - 23307 => 133, 23308 => 134, 23422 => 135, 23466 => 136, - 23505 => 137, 23534 => 138, 23545 => 139, 23650 => 140, - 23655 => 141, 23685 => 142, 23702 => 143, 23727 => 144, - 23730 => 145, 23750 => 146, 23754 => 147, 23770 => 148, - 23771 => 149, 23773 => 150, 23774 => 151, 23782 => 152, - 23784 => 153, 23786 => 154, 23838 => 155, 23964 => 156, - 24084 => 129, 24237 => 157, 24265 => 158, 24301 => 159, - 24370 => 160, 24665 => 161, 24786 => 162, 24902 => 163, - 24903 => 164, 24913 => 165, 25666 => 125, 25793 => 166, - 25820 => 167, 25945 => 168, 26120 => 169, 26196 => 170, - 26268 => 171, 26360 => 172, 26396 => 173, 26397 => 174, - 26624 => 175, 26634 => 176, 26666 => 177, 26702 => 178, - 26716 => 179, 26763 => 180, 26884 => 181, 26939 => 182, - 27168 => 183, 27174 => 184, 27182 => 185, 27268 => 186, - 27326 => 187, 27354 => 188, 27408 => 189, 27423 => 190, - 27433 => 191, 27491 => 192, 27587 => 193, 27626 => 194, - 27638 => 195, 27743 => 196, 27772 => 197, 27799 => 198, - 27819 => 199, 27865 => 200, 27868 => 201, 27903 => 202, - 27987 => 203, 28030 => 204, 28050 => 205, 28065 => 206, - 28074 => 207, 28108 => 208, 28181 => 209, 28182 => 210, - 28183 => 211, 28187 => 212, 28285 => 213, 28336 => 214, - 28339 => 215, 28405 => 216, 28462 => 217, 28601 => 218, - 28640 => 219, 28748 => 220, 28827 => 221, 28855 => 222, - 28856 => 223, # Clone: 28856 => 507, - 28858 => 224, 28960 => 132, 28964 => 127, 29033 => 225, - 29256 => 226, 29299 => 227, 29302 => 228, 29370 => 229, - 29432 => 230, 29548 => 231, 29749 => 232, 29756 => 233, - 29774 => 234, 29931 => 235, 29984 => 236, 30013 => 237, - 30016 => 238, 30026 => 239, 30027 => 240, 30049 => 241, - 30058 => 242, 30232 => 243, 30237 => 244, 30240 => 245, - 30242 => 246, 30265 => 247, 30327 => 248, 30330 => 249, - 30360 => 250, 30376 => 251, 30382 => 252, 30421 => 253, - 30429 => 254, 30452 => 255, 30480 => 256, 30522 => 257, - 30617 => 258, 30621 => 259, 30628 => 260, 30629 => 261, - 30668 => 262, 30678 => 263, 30685 => 264, 30736 => 265, - 30785 => 266, 30818 => 267, 30835 => 268, 30844 => 269, - 30977 => 270, 30985 => 271, 31061 => 272, 31120 => 273, - 31149 => 274, 31174 => 275, 31240 => 276, 31241 => 277, - 31294 => 278, 31350 => 279, 31368 => 280, 31420 => 281, - 31469 => 282, 31508 => 283, 31554 => 284, 31617 => 285, - 31619 => 286, 31690 => 287, 31706 => 288, 31708 => 289, - 31746 => 290, 31747 => 291, 31748 => 292, 31784 => 293, - 31785 => 294, 31841 => 295, 31882 => 296, 31926 => 297, - 31976 => 298, 32053 => 299, 32055 => 300, 32088 => 301, - 32090 => 302, 32109 => 303, 32115 => 304, 32143 => 305, - 32167 => 306, 32171 => 307, 32192 => 308, 32227 => 309, - 32228 => 310, 32234 => 311, 32291 => 312, 32307 => 313, - 32334 => 314, 32353 => 315, 32365 => 316, 32403 => 317, - 32432 => 318, 32467 => 319, 32468 => 320, 32580 => 321, - 32626 => 322, 32674 => 323, 32687 => 324, 32712 => 325, - 32847 => 326, 32887 => 327, 32921 => 328, 32942 => 329, - 32965 => 330, 32981 => 331, 32999 => 332, 33019 => 333, - 33076 => 334, 33134 => 335, 33158 => 336, 33161 => 337, - 33197 => 338, 33239 => 339, 33389 => 340, 33395 => 341, - 33397 => 342, 33442 => 343, 33449 => 344, 33459 => 345, - 33472 => 346, 33642 => 347, 33648 => 348, 33649 => 349, - 33654 => 350, 33678 => 351, 33725 => 352, 33799 => 353, - 33820 => 354, 33835 => 355, 33848 => 356, 33851 => 357, - 33877 => 358, 33884 => 359, 33974 => 360, 34028 => 361, - 34066 => 362, 34149 => 363, 34154 => 364, 34193 => 365, - 34279 => 366, 34320 => 367, 34331 => 368, 34359 => 369, - 34407 => 370, 34408 => 371, 34447 => 372, 34453 => 373, - 34477 => 374, # Clone: 34477 => 459, - 34486 => 375, 34528 => 376, 34544 => 377, 34545 => 378, - 34563 => 379, 34570 => 380, 34585 => 381, 34629 => 382, - 34673 => 383, 34684 => 384, 34695 => 385, 34816 => 386, - 34882 => 387, 34930 => 388, 34946 => 389, 34995 => 390, - 35029 => 391, 35037 => 392, 35157 => 393, 35241 => 394, - 35284 => 395, # Clone: 35284 => 466, - 35388 => 396, 35446 => 397, 35454 => 398, 35455 => 399, - 35456 => 400, 35468 => 401, 35491 => 402, 35518 => 403, - 35626 => 404, 35664 => 405, 35665 => 406, 35668 => 407, - 35729 => 408, 35730 => 409, 35731 => 410, 35796 => 411, - 35822 => 412, 35823 => 413, 35838 => 414, 35879 => 415, - # Clone: 35879 => 616, - 35886 => 416, 35971 => 417, 36021 => 418, 36078 => 419, - 36101 => 420, 36135 => 421, 36147 => 422, 36197 => 423, - 36219 => 424, 36241 => 425, 36242 => 426, 36292 => 427, - 36296 => 428, 36333 => 429, 36622 => 430, 36623 => 431, - 36628 => 432); -} - -1; diff --git a/gradle/documentation/changes-to-html/changes2html.py b/gradle/documentation/changes-to-html/changes2html.py new file mode 100755 index 000000000000..cfd151b28c20 --- /dev/null +++ b/gradle/documentation/changes-to-html/changes2html.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Transforms Solr's CHANGELOG.md into Changes.html + +Input is from CHANGELOG.md, output is to STDOUT +""" + +import sys +import re +from datetime import datetime +from pathlib import Path + + +class ChangelogParser: + """Parse CHANGELOG.md generated by logchange""" + + def __init__(self): + self.title = "Solr Changelog" + self.releases = [] + + def parse(self, content): + """Parse CHANGELOG.md content""" + lines = content.split('\n') + + current_release = None + current_section = None + current_items = [] + self.preamble = None # Store preamble text + + i = 0 + while i < len(lines): + line = lines[i] + + # Skip comments at the start + if line.strip().startswith(''): + i += 1 + continue + + # Extract preamble (text before first release) + if not current_release and not self.preamble and line.strip() and not line.startswith('['): + self.preamble = line.strip() + i += 1 + continue + + # Match release header: [9.9.0] - 2025-07-24 + match = re.match(r'^\[(\d+(?:\.\d+)*)\]\s*-\s*(.+)$', line) + if match: + # Save previous release if exists + if current_release: + if current_section and current_items: + current_release['sections'].append({ + 'name': current_section, + 'items': current_items + }) + self.releases.append(current_release) + + version = match.group(1) + date = match.group(2).strip() + current_release = { + 'version': version, + 'date': date, + 'sections': [] + } + current_section = None + current_items = [] + i += 1 + continue + + # Match section header: ### Added (9 changes) + match = re.match(r'^###\s+(\w+(?:\s+\w+)*)\s*(?:\(\d+\s+changes?\))?', line) + if match and current_release is not None: + # Save previous section + if current_section and current_items: + current_release['sections'].append({ + 'name': current_section, + 'items': current_items + }) + + current_section = match.group(1) + current_items = [] + i += 1 + continue + + # Match list items: - item text + if line.startswith('- ') and current_release is not None: + # Collect multi-line item + item_text = line[2:] # Remove leading '- ' + i += 1 + + # Collect continuation lines + while i < len(lines): + next_line = lines[i] + if next_line.startswith('- ') or re.match(r'^###', next_line) or re.match(r'^\[', next_line): + break + if next_line.strip(): + item_text += ' ' + next_line.strip() + i += 1 + + current_items.append(item_text) + continue + + i += 1 + + # Save last release and section + if current_release: + if current_section and current_items: + current_release['sections'].append({ + 'name': current_section, + 'items': current_items + }) + self.releases.append(current_release) + + +class HTMLGenerator: + """Generate HTML from parsed changelog""" + + JIRA_URL_PREFIX = 'https://issues.apache.org/jira/browse/' + GITHUB_PR_PREFIX = 'https://github.com/apache/solr/pull/' + + def __init__(self, title="Solr Changelog"): + self.title = title + self.first_relid = None + self.second_relid = None + + def extract_issue_from_text(self, text): + """ + Extract the first JIRA/GitHub issue from markdown text. + Returns (issue_link_html, text_without_issue, authors) + """ + # Extract JIRA issue: [SOLR-123](url) + jira_match = re.search(r'\[([A-Z]+-\d+)\]\(https://issues\.apache\.org/jira/browse/\1\)', text) + if jira_match: + issue_id = jira_match.group(1) + issue_html = f'{issue_id}' + text_before = text[:jira_match.start()] + text_after = text[jira_match.end():] + # Combine the text and remove markdown links from issue part + clean_text = (text_before + text_after).strip() + return issue_html, clean_text, None + + # Extract GitHub PR: [PR#123](url) + pr_match = re.search(r'\[PR#(\d+)\]\(https://github\.com/apache/solr/pull/\1\)', text) + if pr_match: + pr_num = pr_match.group(1) + issue_html = f'PR#{pr_num}' + text_before = text[:pr_match.start()] + text_after = text[pr_match.end():] + clean_text = (text_before + text_after).strip() + return issue_html, clean_text, None + + # Extract GITHUB: [GITHUB#123](url) + gh_match = re.search(r'\[GITHUB#(\d+)\]\(https://github\.com/apache/solr/issues/\1\)', text) + if gh_match: + gh_num = gh_match.group(1) + issue_html = f'GITHUB#{gh_num}' + text_before = text[:gh_match.start()] + text_after = text[gh_match.end():] + clean_text = (text_before + text_after).strip() + return issue_html, clean_text, None + + return None, text, None + + def extract_authors(self, text): + """Extract authors from trailing parentheses""" + # Match (author1) (author2) ... at the end + match = re.search(r'\s*(\([^)]+(?:\)\s*\([^)]+)*\))\s*$', text) + if match: + authors_text = match.group(1) + text_without_authors = text[:match.start()].strip() + + # Parse individual authors + authors = re.findall(r'\(([^)]+)\)', authors_text) + authors_list = [] + for author_group in authors: + # Split by comma or "and" + for author in re.split(r',\s*|\s+and\s+', author_group): + author = author.strip() + if author: + authors_list.append(author) + + return authors_list, text_without_authors + return None, text + + def format_changelog_item(self, item_text): + """ + Format a changelog item from markdown to HTML + Format: [ISSUE](url) description (author1) (author2) + Output: ISSUE: description
(authors) + """ + # First extract the issue + issue_html, text_after_issue, _ = self.extract_issue_from_text(item_text) + + if not issue_html: + # No issue found, just linkify and return + return self.linkify_remaining_text(item_text) + + # Extract authors + authors_list, description = self.extract_authors(text_after_issue) + + # Clean up the description + description = description.strip() + # Remove leading punctuation if any + description = re.sub(r'^[:\s]+', '', description).strip() + + # Build the HTML + html = f'{issue_html}: {self.escape_html(description)}' + + if authors_list: + authors_str = ', '.join(authors_list) + html += f'
({self.escape_html(authors_str)})' + + return html + + def linkify_remaining_text(self, text): + """Linkify URLs and remaining JIRA references""" + text = self.escape_html(text) + + # Link remaining JIRA issues + text = re.sub( + r'([A-Z]+-\d+)', + lambda m: f'{m.group(1)}', + text + ) + + # Linkify URLs + text = re.sub( + r'(?])(https?://[^\s\)]+)', + lambda m: f'{m.group(1)}', + text + ) + + return text + + def convert_markdown_links(self, text): + """ + Convert markdown links [text](url) to HTML links text + Also linkifies plain HTTP/HTTPS URLs + Also escapes HTML in plain text portions + """ + placeholders = {} + placeholder_counter = [0] + + def protect_with_placeholder(content): + placeholder = f"__PLACEHOLDER_{placeholder_counter[0]}__" + placeholders[placeholder] = content + placeholder_counter[0] += 1 + return placeholder + + # Pattern: [text](url) + def replace_markdown_link(match): + link_text = match.group(1) + link_url = match.group(2) + html_link = f'{self.escape_html(link_text)}' + return protect_with_placeholder(html_link) + + # Replace all markdown links first + result = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', replace_markdown_link, text) + + # Now handle plain URLs + def replace_url(match): + url = match.group(1) + html_link = f'{url}' + return protect_with_placeholder(html_link) + + # Match HTTP/HTTPS URLs not already in links + result = re.sub(r'(?])(https?://[^\s\)]+)', replace_url, result) + + # Escape HTML in remaining text + result = self.escape_html(result) + + # Restore the protected tags + for placeholder, tag in placeholders.items(): + result = result.replace(placeholder, tag) + + return result + + def escape_html(self, text): + """Escape HTML special characters""" + text = text.replace('&', '&') + text = text.replace('<', '<') + text = text.replace('>', '>') + text = text.replace('"', '"') + return text + + def generate_header(self, preamble=None): + """Generate HTML header""" + first_relid_regex = re.escape(self.first_relid or 'trunk') + first_relid_regex = first_relid_regex.replace('\\', '\\\\') + second_relid_regex = re.escape(self.second_relid or '') + second_relid_regex = second_relid_regex.replace('\\', '\\\\') + + newer_version_regex = f"^(?:{first_relid_regex}" + if self.second_relid: + newer_version_regex += f"|{second_relid_regex}" + newer_version_regex += ")" + + html = f''' + + + + Apache Solr Release Notes + + + + + + + + +

Apache Solr Release Notes

+ +
+ +''' + # Add preamble if present + if preamble: + # Convert markdown links to HTML links + preamble_html = self.convert_markdown_links(preamble) + html += f'

{preamble_html}

\n\n' + + return html + + def generate_releases(self, releases): + """Generate HTML for releases""" + html = [] + relcnt = 0 + header = 'h2' + subheader = 'h3' + + for release in releases: + if relcnt == 2: + header = 'h3' + subheader = 'h4' + html.append('

Older Releases

\n') + html.append('
\n') + + version = release.get('version') + date = release.get('date', '') + sections = release.get('sections', []) + + if version: + relcnt += 1 + relid = f'v{version}'.replace(' ', '_').lower() + + html.append(f'<{header}>') + html.append(f'') + html.append('Release ' if version else '') + html.append(f'{self.escape_html(version)} ') + if date: + html.append(f'[{self.escape_html(date)}]') + html.append('') + html.append(f'\n') + html.append(f'
    \n') + + # Render sections + licnt = 0 + for section in sections: + section_name = section.get('name', '') + items = section.get('items', []) + + if section_name: + sectid = section_name.lower().replace(' ', '_') + num_items = len(items) + num_str = f"({num_items})" + + html.append(f'
  • ') + html.append(f'{self.escape_html(section_name)}') + html.append(f'   {num_str}\n') + licnt += 1 + + html.append(f'
      \n') + + for item in items: + # Format the item properly + item_html = self.format_changelog_item(item) + + html.append(f'
    • {item_html}
      • \n') + + html.append('
    \n') + + html.append('
\n') + + if relcnt > 2: + html.append('
\n') + + return ''.join(html) + + def generate(self, releases, title, preamble=None): + """Generate complete HTML""" + self.title = title or "Solr Changelog" + + # Determine first and second release IDs for collapsing + if releases: + self.first_relid = f'v{releases[0].get("version", "trunk")}'.replace(' ', '_').lower() + if len(releases) > 1: + self.second_relid = f'v{releases[1].get("version", "trunk")}'.replace(' ', '_').lower() + else: + self.second_relid = self.first_relid + + html_parts = [ + self.generate_header(preamble), + self.generate_releases(releases), + '\n\n' + ] + + return ''.join(html_parts) + + +def main(): + """Main entry point""" + if len(sys.argv) < 2: + # Try to read from CHANGELOG.md in current directory + changelog_file = Path('CHANGELOG.md') + if not changelog_file.exists(): + print("Usage: changes2html.py ", file=sys.stderr) + sys.exit(1) + else: + changelog_file = Path(sys.argv[1]) + + if not changelog_file.exists(): + print(f"Error: {changelog_file} not found", file=sys.stderr) + sys.exit(1) + + # Read changelog + with open(changelog_file, 'r', encoding='utf-8') as f: + content = f.read() + + # Parse + parser = ChangelogParser() + parser.parse(content) + + # Generate HTML + generator = HTMLGenerator() + html = generator.generate(parser.releases, parser.title, parser.preamble) + + # Output + print(html) + + +if __name__ == '__main__': + main() From 6d34500680b000aeeebc0eaa4e99ae41e342c6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 03:19:26 +0200 Subject: [PATCH 30/64] Note about not using upgrade_notes type --- changelog/logchange-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 957a99ea01ed..849bee031099 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -19,6 +19,7 @@ changelog: heading: This file lists Solr's raw release notes with details of every change to Solr. Most people will find the solr-upgrade-notes.adoc file more approachable. [https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc](https://github.com/apache/solr/blob/main/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc) entryTypes: # you can define custom entry types + # Please do not use upgrade_notes type. - key: upgrade_notes order: 1 - key: added @@ -50,6 +51,7 @@ changelog: dependency_update: Dependency Upgrades security: Security other: Other + # Please do not use upgrade_notes type. upgrade_notes: Upgrade Notes number_of_changes: singular: change From 6cc1684a121ebf86f258734406ff904f1e1921ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 03:42:21 +0200 Subject: [PATCH 31/64] Escape only < and > --- dev-tools/scripts/changes2logchange.py | 6 +++--- gradle/documentation/changes-to-html/changes2html.py | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py index 18f9d64e125b..dabd6cd92d05 100755 --- a/dev-tools/scripts/changes2logchange.py +++ b/dev-tools/scripts/changes2logchange.py @@ -542,9 +542,9 @@ def _parse_single_entry(self, entry_text: str, change_type: str) -> Optional[Cha # This joins multi-line formatted text into a single coherent paragraph description = re.sub(r'\s+', ' ', description) - # Escape HTML entities to prevent markdown rendering issues - # This converts <, >, &, etc. to <, >, & for safe markdown - description = html.escape(description) + # Escape HTML angle brackets to prevent markdown rendering issues + # Only escape < and > to avoid breaking markdown links and quotes + description = description.replace('<', '<').replace('>', '>') if not description: return None diff --git a/gradle/documentation/changes-to-html/changes2html.py b/gradle/documentation/changes-to-html/changes2html.py index cfd151b28c20..d576d99dc4c4 100755 --- a/gradle/documentation/changes-to-html/changes2html.py +++ b/gradle/documentation/changes-to-html/changes2html.py @@ -293,11 +293,10 @@ def replace_url(match): return result def escape_html(self, text): - """Escape HTML special characters""" - text = text.replace('&', '&') + """Escape HTML angle brackets to prevent rendering issues""" + # Only escape < and > to avoid breaking markdown links and quotes text = text.replace('<', '<') text = text.replace('>', '>') - text = text.replace('"', '"') return text def generate_header(self, preamble=None): From 297acb516147b858b654b7028d2edfb13272c96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 08:46:36 +0200 Subject: [PATCH 32/64] Dummy changelog file to keep build from failing --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000000..17d4d0c05ce3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ + + + + + + + + + +DUMMY changelog file. + +We are in the process of migrating from CHANGES.txt to a structured approach to changelog generation. See [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) for details. + +[unreleased] +------------ + +### Added (1 change) + +- Dummy issue [SOLR-123](https://issues.apache.org/jira/browse/SOLR-123) (janhoy) + +[9.9.0] - 2025-07-24 +-------------------- + +### Added (1 change) + +- Dummy issue for release 9.9 [SOLR-124](https://issues.apache.org/jira/browse/SOLR-124) (janhoy) From 2398f39df9008303f12b39cecee9ddb3b9552494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 09:38:53 +0200 Subject: [PATCH 33/64] Document migratino tool in dev-docs, including new option to convert individual entries from stdin --- dev-docs/changelog.adoc | 30 +++++- dev-tools/scripts/changes2logchange.py | 129 ++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 50fc2f22f650..48bc96690710 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -116,7 +116,35 @@ links: url: https://issues.apache.org/jira/browse/SOLR-3333 ---- -== 7. Further Reading +== 7. Migration tool + +There is a migration tool in `dev-tools/scripts/changes2logchange.py`. +It will bulk convert the entire `solr/CHANGES.txt` file to files in the `changelog/` folder and versioned sub folders. + +The tool can also convert a single changelog entry reading `stdin` as follows: + +```bash +cat << EOF | python3 dev-tools/scripts/changes2logchange.py - +* SOLR-17960: Removed TikaLanguageIdentifierUpdateProcessor. + Use LangDetectLanguageIdentifierUpdateProcessor or + OpenNLPLangDetectUpdateProcessor instead. (janhoy) +EOF +``` + +which will output to `stdout`: + +``` +title: Removed TikaLanguageIdentifierUpdateProcessor. Use LangDetectLanguageIdentifierUpdateProcessor + or OpenNLPLangDetectUpdateProcessor instead. +type: other +authors: +- name: janhoy +links: +- name: SOLR-17960 + url: https://issues.apache.org/jira/browse/SOLR-17960 +``` + +== 8. Further Reading * xref:https://github.com/logchange/logchange[Logchange web page] * xref:https://keepachangelog.com/en/1.1.0/[keepachangelog.com website] diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py index dabd6cd92d05..20ef3cfe6582 100755 --- a/dev-tools/scripts/changes2logchange.py +++ b/dev-tools/scripts/changes2logchange.py @@ -758,6 +758,128 @@ def _print_summary(self): print("="*60) +class StdinProcessor: + """Process individual changelog entries from stdin and output YAML to stdout.""" + + @staticmethod + def process(): + """ + Read from stdin, parse individual changelog entries, and output YAML. + + Ignores headers and nested structure. + Outputs YAML entries separated by '----' YAML separator. + """ + import sys + + # Read all lines from stdin + lines = sys.stdin.readlines() + + entries_yaml = [] + i = 0 + + while i < len(lines): + line = lines[i] + + # Skip empty lines and header lines (lines with only dashes or equals) + if not line.strip() or re.match(r'^[-=\s]+$', line): + i += 1 + continue + + # Check if this line starts a changelog entry (bullet point) + if line.strip().startswith('*') or line.strip().startswith('-'): + # Collect the full entry (may span multiple lines) + entry_text = line.strip()[1:].strip() # Remove bullet and leading spaces + + # Continue reading continuation lines + i += 1 + while i < len(lines): + next_line = lines[i] + # If the next line is another entry or empty, stop collecting + if (next_line.strip().startswith('*') or + next_line.strip().startswith('-') or + re.match(r'^[-=\s]+$', next_line) or + not next_line.strip()): + break + # Add to entry text + entry_text += ' ' + next_line.strip() + i += 1 + + # Parse the entry to a ChangeEntry + entry = EntryParser.parse_entry_line(entry_text) + if entry: + # Serialize to YAML + yaml_dict = { + 'title': entry.title, + 'type': entry.change_type, + } + if entry.authors: + yaml_dict['authors'] = [{'name': a.name} for a in entry.authors] + if entry.links: + yaml_dict['links'] = [ + {'name': link.name, 'url': link.url} + for link in entry.links + ] + + yaml_str = yaml.dump(yaml_dict, default_flow_style=False, sort_keys=False, allow_unicode=True) + entries_yaml.append(yaml_str.rstrip()) + else: + i += 1 + + # Output entries separated by YAML separators + for i, yaml_entry in enumerate(entries_yaml): + if i > 0: + print('----') + print(yaml_entry, end='') + if yaml_entry and not yaml_entry.endswith('\n'): + print() + + +class EntryParser: + """Parse a single changelog entry line.""" + + @staticmethod + def parse_entry_line(text: str) -> Optional[ChangeEntry]: + """ + Parse a single changelog entry line. + + Format: [ISSUE-ID: ]description (author1) (author2) ... + """ + if not text.strip(): + return None + + # Extract issue links + links = IssueExtractor.extract_issues(text) + + # Remove issue IDs from text + for link in links: + # Remove markdown link format [ID](url) + text = re.sub(rf'\[{re.escape(link.name)}\]\([^)]+\)', '', text) + # Remove plain text issue IDs + text = re.sub(rf'{re.escape(link.name)}\s*:?\s*', '', text) + + text = text.strip() + + # Extract authors + text, authors = AuthorParser.parse_authors(text) + text = text.strip() + + # Escape HTML angle brackets + text = text.replace('<', '<').replace('>', '>') + + if not text: + return None + + # Default to 'other' type + change_type = 'other' + + return ChangeEntry( + title=text, + change_type=change_type, + authors=authors, + links=links, + ) + + def main(): """Main entry point.""" import argparse @@ -767,7 +889,7 @@ def main(): ) parser.add_argument( "changes_file", - help="Path to the CHANGES.txt file to migrate" + help="Path to the CHANGES.txt file to migrate. Use '-' to read individual changelog entries from stdin and output YAML to stdout" ) parser.add_argument( "-o", "--output-dir", @@ -782,6 +904,11 @@ def main(): args = parser.parse_args() + # Handle stdin/stdout mode + if args.changes_file == '-': + StdinProcessor.process() + return + if not os.path.exists(args.changes_file): print(f"Error: CHANGES.txt file not found: {args.changes_file}", file=sys.stderr) sys.exit(1) From 3e494b655cbb7c3d9aaf97a9f18a130f88dfaaad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 16:05:16 +0200 Subject: [PATCH 34/64] s/disallowed/deprecated/ --- .github/workflows/validate-changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 2ea4c41bb74e..c518ccb3dcde 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -33,7 +33,7 @@ jobs: CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) if echo "$CHANGED_FILES" | grep -q "^solr/CHANGES\.txt$"; then - echo "::error::Use of solr/CHANGES.txt is disallowed. Please create a changelog yaml file instead." + echo "::error::Use of solr/CHANGES.txt is deprecated. Please create a changelog yaml file instead." echo "" echo "Instead of editing CHANGES.txt, please:" echo "1. Run: ./gradlew generateChangeYaml" From 6bdfe64f5f7528c8d2f32d04c8584960ac8c3850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 16:20:15 +0200 Subject: [PATCH 35/64] Add a README (tmp) in the changelog/ folder --- CHANGELOG.md | 2 +- changelog/README.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d4d0c05ce3..1d34ba1a95b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ DUMMY changelog file. -We are in the process of migrating from CHANGES.txt to a structured approach to changelog generation. See [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) for details. +We are in the process of migrating from CHANGES.txt to a structured approach to changelog generation. See [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) as well as [dev-docs/changelog.md](dev-docs/changelog.md) for details. [unreleased] ------------ diff --git a/changelog/README.md b/changelog/README.md new file mode 100644 index 000000000000..23a72bf74b13 --- /dev/null +++ b/changelog/README.md @@ -0,0 +1,5 @@ +# New changelog process + +We are in the process of migrating to a new way of managing our changelog. Please see [dev-docs/changelog.md](../dev-docs/changelog.md) for details. + +In a transition period it is still possible to merge your changelog entry to `solr/CHANGES.txt`, but on Wednesday October 29th this will not longer be the case. From bcfb568c386956cb0f9c585989fe80c80da1b7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 18:08:58 +0200 Subject: [PATCH 36/64] Include CHANGELOG.md in tarball --- solr/packaging/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 2c21f928d0b5..a6f0cdb259c8 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -86,7 +86,7 @@ distributions { include "bin/**" include "licenses/**" exclude "licenses/README.committers.txt" - include "CHANGES.txt" + include "CHANGELOG.md" }) from(projectDir, { From a1fad3c722a7e00df00462cb9036389939f9f4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 18:09:39 +0200 Subject: [PATCH 37/64] Remove CHANGES.txt mention from addDepsToChanges.py --- dev-tools/scripts/addDepsToChanges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py index 1e1ab152f667..a53066b0a757 100755 --- a/dev-tools/scripts/addDepsToChanges.py +++ b/dev-tools/scripts/addDepsToChanges.py @@ -119,7 +119,7 @@ def get_prev_release_tag(ver): def read_config(): - parser = argparse.ArgumentParser(description='Adds dependency changes section to CHANGES.txt.') + parser = argparse.ArgumentParser(description='Adds changelog entries in changelog/ folder') parser.add_argument('--version', type=Version.parse, help='Solr version to add changes to', required=True) parser.add_argument('--user', default='solrbot', help='Git user to get changes for. Defaults to solrbot') newconf = parser.parse_args() @@ -204,7 +204,7 @@ def sort_entries(entries): def main(): - if not os.path.exists('solr/CHANGES.txt'): + if not os.path.exists('CHANGELOG.md'): sys.exit("Tool must be run from the root of a source checkout.") newconf = read_config() prev_tag = get_prev_release_tag(newconf.version) From a26cfd12f4bdc50ee85d0bc79efd471cdbdb9557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 18:09:54 +0200 Subject: [PATCH 38/64] Remove sentence from dev-docs/git.adoc --- dev-docs/git.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-docs/git.adoc b/dev-docs/git.adoc index a21b1313eb74..fe28ec0b0d49 100644 --- a/dev-docs/git.adoc +++ b/dev-docs/git.adoc @@ -229,5 +229,3 @@ $ ./gradlew check -x test # Run tests if you need to $ git show HEAD # This will show you the commit you are about to push, make sure it looks right $ git push apache branch_9x ---- - -There has been some issue with Solr's CHANGES.txt file "cherry picking" all of the changes for trunk, so check this file especially. From 6e965bbcd8588bbb89d5c966b0bdce38dc7dcb78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 19:33:02 +0200 Subject: [PATCH 39/64] Fix ExternalPaths.java Some wording changes --- changelog/unreleased/.keep | 0 solr/licenses/README.committers.txt | 2 +- .../modules/upgrade-notes/pages/solr-upgrade-notes.adoc | 4 ++-- .../src/java/org/apache/solr/util/ExternalPaths.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 changelog/unreleased/.keep diff --git a/changelog/unreleased/.keep b/changelog/unreleased/.keep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/solr/licenses/README.committers.txt b/solr/licenses/README.committers.txt index 17dab04632b8..2942a6c8953e 100644 --- a/solr/licenses/README.committers.txt +++ b/solr/licenses/README.committers.txt @@ -29,7 +29,7 @@ what changed with respect to licensing in the commit diff. --- -Any changes made to this directory should be noted in CHANGES.txt, +Any changes made to this directory should be noted in the changelog, along with the specific version information. If the version is a "snapshot" of another Apache project, include the SVN revision number. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc index 890a2a98ffa9..061aea238e4a 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/solr-upgrade-notes.adoc @@ -27,7 +27,7 @@ These notes highlight the important changes that may impact the largest number o It is not a comprehensive list of all changes to Solr in any release. When planning your Solr upgrade, consider the customizations to -your system and review the {solr-javadocs}/changes/Changes.html[`CHANGES.txt`] +your system and review the {solr-javadocs}/changes/Changes.html[`CHANGELOG.md`] file found in your Solr package. That file includes all the changes and updates that may affect your existing implementation. @@ -67,7 +67,7 @@ Rolling upgrades from Solr 8 to Solr 9 require first upgrading the cluster to ve == Upgrading from Pre-8.x Versions -Users upgrading from versions prior to 8.x are strongly encouraged to consult {solr-javadocs}/changes/Changes.html[`CHANGES.txt`] for the details of _all_ changes since the version they are upgrading from. +Users upgrading from versions prior to 8.x are strongly encouraged to consult {solr-javadocs}/changes/Changes.html[`CHANGELOG.md`] for the details of _all_ changes since the version they are upgrading from. The upgrade from 7.x to 8.0 introduced several *major* changes that you should be aware of before upgrading. Please do a thorough review of the section xref:major-changes-in-solr-8.adoc[] before starting your upgrade. diff --git a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java index 9d5f82a24089..f38c01bc3838 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java +++ b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java @@ -82,7 +82,7 @@ static Path determineSourceHome() { } Path base = file.toAbsolutePath(); - while (!Files.exists(base.resolve("solr/CHANGES.txt")) && null != base) { + while (!Files.exists(base.resolve("CHANGELOG.md")) && null != base) { base = base.getParent(); } return (null == base) ? null : base.resolve("solr/").toAbsolutePath(); From 2e1711f2434b1bd705c30b0005e0e86a16d688fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 19:35:07 +0200 Subject: [PATCH 40/64] Remove unused script releasedJirasRegex.py --- dev-tools/scripts/README.md | 18 ----- dev-tools/scripts/releasedJirasRegex.py | 91 ------------------------- 2 files changed, 109 deletions(-) delete mode 100755 dev-tools/scripts/releasedJirasRegex.py diff --git a/dev-tools/scripts/README.md b/dev-tools/scripts/README.md index 438be543fa3b..e7d781d91b16 100644 --- a/dev-tools/scripts/README.md +++ b/dev-tools/scripts/README.md @@ -106,24 +106,6 @@ of the other tools in this folder. -h, --help show this help message and exit -l LUCENE_VERSION Optional lucene version. By default will read gradle/libs.versions.toml -### releasedJirasRegex.py - -Pulls out all JIRAs mentioned at the beginning of bullet items -under the given version in the given CHANGES.txt file -and prints a regular expression that will match all of them - - usage: releasedJirasRegex.py [-h] version changes - - Prints a regex matching JIRAs fixed in the given version by parsing the given - CHANGES.txt file - - positional arguments: - version Version of the form X.Y.Z - changes CHANGES.txt file to parse - - optional arguments: - -h, --help show this help message and exit - ### reproduceJenkinsFailures.py usage: reproduceJenkinsFailures.py [-h] [--no-git] [--iters N] URL diff --git a/dev-tools/scripts/releasedJirasRegex.py b/dev-tools/scripts/releasedJirasRegex.py deleted file mode 100755 index 77780ffcaa63..000000000000 --- a/dev-tools/scripts/releasedJirasRegex.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os -sys.path.append(os.path.dirname(__file__)) -from scriptutil import * -import argparse -import re - -# Pulls out all JIRAs mentioned at the beginning of bullet items -# under the given version in the given CHANGES.txt file -# and prints a regular expression that will match all of them -# -# Caveat: In ancient versions (Solr v1.1 and older), -# does not find Bugzilla bugs or JIRAs not mentioned at the beginning of -# bullets or numbered entries. -# -def print_released_jiras_regex(version, filename): - release_boundary_re = re.compile(r'\s*====*\s+(.*)\s+===') - version_re = re.compile(r'%s(?:$|[^-])' % version) - bullet_re = re.compile(r'\s*(?:[-*]|\d+\.(?=(?:\s|(?:SOLR)-)))(.*)') - jira_ptn = r'(?:SOLR)-\d+' - jira_re = re.compile(jira_ptn) - jira_list_ptn = r'(?:[:,/()\s]*(?:%s))+' % jira_ptn - jira_list_re = re.compile(jira_list_ptn) - more_jiras_on_next_line_re = re.compile(r'%s\s*,\s*$' % jira_list_ptn) # JIRA list with trailing comma - under_requested_version = False - requested_version_found = False - more_jiras_on_next_line = False - solr_jiras = [] - with open(filename, 'r') as changes: - for line in changes: - version_boundary = release_boundary_re.match(line) - if version_boundary is not None: - if under_requested_version: - break # No longer under the requested version - stop looking for JIRAs - else: - if version_re.search(version_boundary.group(1)): - under_requested_version = True # Start looking for JIRAs - requested_version_found = True - else: - if under_requested_version: - bullet_match = bullet_re.match(line) - if more_jiras_on_next_line or bullet_match is not None: - content = line if bullet_match is None else bullet_match.group(1) - jira_list_match = jira_list_re.match(content) - if jira_list_match is not None: - jira_match = jira_re.findall(jira_list_match.group(0)) - for jira in jira_match: - solr_jiras.append(jira.rsplit('-', 1)[-1]) - more_jiras_on_next_line = more_jiras_on_next_line_re.match(content) - if not requested_version_found: - raise Exception('Could not find %s in %s' % (version, filename)) - print() - if (len(solr_jiras) == 0): - print('(No JIRAs => no regex)', end='') - else: - print(r'SOLR-(?:%s)\b' % '|'.join(solr_jiras), end='') - print() - -def read_config(): - parser = argparse.ArgumentParser( - description='Prints a regex matching JIRAs fixed in the given version by parsing the given CHANGES.txt file') - parser.add_argument('version', type=Version.parse, help='Version of the form X.Y.Z') - parser.add_argument('changes', help='CHANGES.txt file to parse') - return parser.parse_args() - -def main(): - config = read_config() - print_released_jiras_regex(config.version, config.changes) - -if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - print('\nReceived Ctrl-C, exiting early') From d9a13446aa7db824c79856bd0f2a2a17de28c517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 20:01:25 +0200 Subject: [PATCH 41/64] Update smokeTestRelease.py to require CHANGELOG.md --- dev-tools/scripts/smokeTestRelease.py | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 9ff1e24774c6..729e97823471 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -380,15 +380,19 @@ def testOpenApi(version, openApiDirUrl): raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl) -def testChangesText(dir, version): - "Checks all CHANGES.txt under this dir." - for root, dirs, files in os.walk(dir): # pylint: disable=unused-variable +def testChangelogMd(dir, version): + "Checks CHANGELOG.md file." + changelog_path = os.path.join(dir, 'CHANGELOG.md') - # NOTE: O(N) but N should be smallish: - if 'CHANGES.txt' in files: - fullPath = '%s/CHANGES.txt' % root - #print 'CHECK %s' % fullPath - checkChangesContent(open(fullPath, encoding='UTF-8').read(), version, fullPath, False) + if not os.path.exists(changelog_path): + raise RuntimeError('CHANGELOG.md not found at %s' % changelog_path) + + with open(changelog_path, encoding='UTF-8') as f: + content = f.read() + + # Verify that the changelog contains the current version + if 'v%s' % version not in content and version not in content: + raise RuntimeError('Version %s not found in CHANGELOG.md' % version) reChangesSectionHREF = re.compile('(.*?)', re.IGNORECASE) reUnderbarNotDashHTML = re.compile(r'
  • (\s*(SOLR)_\d\d\d\d+)') @@ -612,10 +616,10 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs): in_solr_folder = [] if isSrc: in_solr_folder.extend(os.listdir(os.path.join(unpackPath, 'solr'))) - is_in_list(in_root_folder, ['LICENSE.txt', 'NOTICE.txt', 'README.md', 'CONTRIBUTING.md']) - is_in_list(in_solr_folder, ['CHANGES.txt', 'README.adoc']) + is_in_list(in_root_folder, ['LICENSE.txt', 'NOTICE.txt', 'README.md', 'CONTRIBUTING.md', 'CHANGELOG.md']) + is_in_list(in_solr_folder, ['README.adoc']) else: - is_in_list(in_root_folder, ['LICENSE.txt', 'NOTICE.txt', 'README.txt', 'CHANGES.txt']) + is_in_list(in_root_folder, ['LICENSE.txt', 'NOTICE.txt', 'README.txt', 'CHANGELOG.md']) if SOLR_NOTICE is None: SOLR_NOTICE = open('%s/NOTICE.txt' % unpackPath, encoding='UTF-8').read() @@ -715,7 +719,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs): os.chdir(unpackPath) - testChangesText('.', version) + testChangelogMd('.', version) def readSolrOutput(p, startupEvent, failureEvent, logFile): @@ -1220,4 +1224,4 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys, try: main() except KeyboardInterrupt: - print('Keyboard interrupt...exiting') \ No newline at end of file + print('Keyboard interrupt...exiting') From 1e9a97b257eeeb6d09e96c5c2458c3a84ec1cbc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 21 Oct 2025 21:28:37 +0200 Subject: [PATCH 42/64] Update releaseWizard changelog handling --- dev-tools/scripts/releaseWizard.yaml | 152 +++++++++++++++------------ 1 file changed, 87 insertions(+), 65 deletions(-) diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index cd10ebc8a15b..6b36f74010e1 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -58,7 +58,7 @@ templates: {% set passed = plus_binding >= 3 and minus < plus_binding %} {% set too_few = plus_binding < 3 %} {% set veto = plus_binding < minus %} - {% set reason = 'too few binding votes' if too_few else 'too many negative votes' if veto else 'unknown' %} + {% set reason = 'too few binding votes' if too_few else 'too many negative votes' if veto else 'unknown' %} vote_macro: | {% macro end_vote_result(plus_binding, plus_other, zero, minus) -%} (( template=vote_logic )) @@ -75,9 +75,9 @@ templates: {% if not passed %} Reason for fail is {{ reason }}. - + {% endif %} - This vote has {% if passed %}PASSED{% else %}FAILED{% endif %} + This vote has {% if passed %}PASSED{% else %}FAILED{% endif %} ---- {%- endmacro %} announce_solr: | @@ -102,7 +102,7 @@ templates: - Please read CHANGES.txt for a full list of {% if is_feature_release %}new features, changes and {% endif %}bugfixes: + Please read CHANGELOG.md for a full list of {% if is_feature_release %}new features, changes and {% endif %}bugfixes: announce_solr_mail: | @@ -116,10 +116,10 @@ templates: (( template=announce_solr_mail_body )) ---- - announce_solr_announce_mail: | + announce_solr_announce_mail: | The template below can be used to announce the Solr release to the `announce@apache.org` mailing list. The email *must* be sent *from your `@apache.org` account*, - and it *can optionally* be signed with your PGP key, using the `--clearsign` option of + and it *can optionally* be signed with your PGP key, using the `--clearsign` option of gpg, as explained in https://www.gnupg.org/gph/en/manual/x135.html. .Mail template @@ -166,12 +166,12 @@ templates: # # Vars can contain global jinja vars or local vars earlier defined (ordered dict) # persist_vars: ['var_name', 'var_name'] # List of variables to persist in TODO state # asciidoc: | -# Some `asciidoc` text to be included in asciidoc guide +# Some `asciidoc` text to be included in asciidoc guide # *instead of* description/post_description # function: my_python_function # Will call the named function for complex tasks # commands: !Commands # A !Commands object holding commands to execute for this todo # root_folder: '{{ git_checkout_folder }}' # path to where commands will run -# commands_text: Introduction text to be displayed just before the commands +# commands_text: Introduction text to be displayed just before the commands # enable_execute: true # Set to false to never offer to run commands automatically # confirm_each_command: true # Set to false to run all commands without prompting # remove_files: ['file1', 'folder2'] # List of files or folders that must be gone @@ -196,7 +196,7 @@ templates: # name: gpg_id # This will be stored in todo state and can be referenced as {{ todo_id.name }} # type: int # if no type is given, a string is stored. Supported types are 'int' # post_description: | -# Some `asciidoc` text (with jinja template support) +# Some `asciidoc` text (with jinja template support) # to be printed *after* commands and user_input is done. # links: # - http://example.com/list/of/links?to&be&displayed @@ -453,7 +453,7 @@ groups: title: Update Ref Guide Metadata for new Minor Branch description: | This will update the "solr/solr-ref-guide/antora.yml" file. - + The file shouldn't change much, but it should pick up new dependency versions, if they have changed. types: - major @@ -557,7 +557,7 @@ groups: description: | This will make sure we are continuously testing the new branch(es) we have created. Consult the JenkinsReleaseBuilds page for more information. - + Job Suffixes to create: {% if release_type == 'major' %} *-{{release_version_major}}.x (The new stable branch) @@ -575,7 +575,7 @@ groups: description: | This will make sure we aren't using resources to test branches no longer in development. Consult the JenkinsReleaseBuilds page for more information. - + Job Suffixes to delete: {% if release_type == 'major' %} *-{{latest_lts_version_major}}.{{latest_lts_version_minor}} (The previous LTS release branch) @@ -665,12 +665,12 @@ groups: - bugfix - !Todo id: dependency_updates_changes - title: Add dependency updates to CHANGES.txt + title: Add dependency updates to CHANGELOG.md description: Bulk add all 'solrbot' dependency updates since last release depends: clean_git_checkout commands: !Commands root_folder: '{{ git_checkout_folder }}' - commands_text: We call out to a helper script that modifies CHANGES.txt, and then commit it. + commands_text: We call out to a helper script that compiles SolrBot changes into `changelog/unreleased` confirm_each_command: true commands: - !Command @@ -680,14 +680,45 @@ groups: cmd: python3 -u dev-tools/scripts/addDepsToChanges.py --user solrbot --version {{ release_version }} tee: true - !Command - cmd: "{{ editor }} solr/CHANGES.txt" - comment: | - Verify that the modifications in CHANGES.txt under the 'Dependency Upgrades' are ok. - Look for duplicates, especially when a committer has manually added a similar entry for the same upgrade. + cmd: git add -u . && git commit -m "Add dependency updates to changelog for {{ release_version }}" && git push + logfile: dependency-changes.log + - !Todo + id: generate_changelog_release + title: Generate CHANGELOG.md for release + description: | + Generate `CHANGELOG.md` by running `logchange release` and `logchange generate`. This will prepare both a `changelog/v{{ release_version }}` folder and the `CHANGELOG.md` file in one commit. + depends: dependency_updates_changes + commands: !Commands + root_folder: '{{ git_checkout_folder }}' + commands_text: Generate CHANGELOG.md + commands: + - !Command + cmd: git checkout {{ release_branch }} stdout: true - !Command - cmd: git add -u . && git commit -m "Add dependency updates to CHANGES for {{ release_version }}" && git push - logfile: dependency-changes.log + cmd: git pull --ff-only + tee: true + - !Command + cmd: "{{ gradle_cmd }} logchangeRelease" + comment: Create `changelog/v{{ release_version }}` folder and move unreleased entries + tee: true + - !Command + cmd: "{{ gradle_cmd }} logchangeGenerate" + comment: Generate `CHANGELOG.md` in repository root + tee: true + - !Command + cmd: git add CHANGELOG.md changelog/v* && git commit -m "Changelog for release v{{ release_version }}" && git push + comment: Commit and push changelog changes + logfile: commit_changelog.log + tee: true + - !Todo + id: persist_changelog_sha + title: Persist the SHA of the changelog commit + description: Store the current git sha + depends: generate_changelog_release + vars: + changelog_sha: {{ current_git_rev }} + persist: changelog_sha - !Todo id: draft_release_notes title: Get a draft of the release notes in place @@ -695,7 +726,7 @@ groups: These are typically edited on the Wiki Clone a page for a previous version as a starting point for your release notes. - Edit the contents of `CHANGES.txt` into a more concise format for public consumption. + Edit the contents of `CHANGELOG.md` into a more concise format for public consumption. Ask on dev@ for input. Ideally the timing of this request mostly coincides with the release branch creation. It's a good idea to remind the devs of this later in the release too. @@ -799,7 +830,7 @@ groups: - !Todo id: build_rc title: Build the release candidate - depends: + depends: - setup_gpg - run_tests vars: @@ -820,7 +851,7 @@ groups: Note that the build will take a long time. To follow the detailed build log, you can tail the log file {{ logfile | default("") }}. confirm_each_command: false - remove_files: + remove_files: - '{{ dist_file_path }}' commands: - !Command @@ -879,7 +910,7 @@ groups: id: verify_staged title: Verify staged artifacts description: | - A lightweight smoke testing which downloads the artifacts from stage + A lightweight smoke testing which downloads the artifacts from stage area and checks hash and signatures, but does not re-run all tests. depends: import_svn vars: @@ -1023,7 +1054,7 @@ groups: there are more positive than negative votes you can techically release the software. However, please review the negative votes and consider a re-spin. - + {% endif %} {%- endif %} {{ end_vote_result(plus_binding,plus_other,zero,minus) }} @@ -1095,10 +1126,10 @@ groups: depends: mv_to_release description: | For every release, we publish the official dockerfile for solr via the methods required by Docker. - + This will start the process, and will ultimately create a PR in https://github.com/docker-library/official-images. This PR creation is automated, you do not need to do more work beyond these steps. - + When that PR is approved and merged, the Official image for this Solr version will be built and available for use. vars: docker_version: '{{ release_version_major }}.{{ release_version_minor }}' @@ -1242,7 +1273,7 @@ groups: The Solr minor branches do not contain up-to-date ref-guide metadata (antora.yml). Instead, they have the information from the last release. This allows us to build the official documentation from these minor branches, instead of from the release tags. - + This means that we can make fixes to older ref-guides without requiring a new release for those versions. depends: tag_release commands: !Commands @@ -1522,7 +1553,7 @@ groups: title: Update the DOAP file description: | Update the Solr DOAP RDF file on the unstable, stable and release branches to - reflect the new versions (note that the website .htaccess file redirects from their + reflect the new versions (note that the website .htaccess file redirects from their canonical URLs to their locations in the Solr Git source repository - see dev-tools/doap/README.txt for more info) commands: !Commands @@ -1586,7 +1617,7 @@ groups: id: announce title: Announce the release description: | - For feature releases, your announcement should describe the main features included + For feature releases, your announcement should describe the main features included in the release. *Send the announce as Plain-text email, not HTML.* This step will generate email templates based on the news files you edited earlier for the website. @@ -1689,52 +1720,43 @@ groups: cmd: git add -u . && git commit -m "Add bugfix version {{ release_version }}" && git push logfile: commit-stable.log - !Todo - id: synchronize_changes - title: Synchronize CHANGES.txt + id: sync_changelog_cherry_pick + title: Cherry-pick changelog to stable and unstable branches description: | - Copy the CHANGES.txt section for this release back to the stable and unstable branches' - CHANGES.txt files, removing any duplicate entries, but only from sections for as-yet - unreleased versions; leave intact duplicate entries for already-released versions. - - There is a script to generate a regex that will match JIRAs fixed in a release: - `releasedJirasRegex.py`. The following examples will print regexes matching all JIRAs - fixed in {{ release_version }}, which can then be used to find duplicates in unreleased - version sections of the corresponding CHANGES.txt files. + Cherry-pick the changelog commit from the release branch to the stable and main branches. + This syncs both CHANGELOG.md and changelog folder changes across all active branches. + + The changelog commit was created in the previous step (generate_changelog_release) and will + be referenced here. Cherry-pick will automatically handle conflict resolution if needed. + depends: generate_changelog_release commands: !Commands root_folder: '{{ git_checkout_folder }}' - commands_text: Synchronize CHANGES.txt + commands_text: Cherry-pick changelog changes to other branches commands: - !Command - cmd: git checkout {{ release_branch }} - comment: Go to release branch - logfile: checkout-release.log + cmd: git checkout {{ stable_branch }} + comment: Checkout stable branch stdout: true - !Command - cmd: python3 -u -B dev-tools/scripts/releasedJirasRegex.py {{ release_version }} solr/CHANGES.txt + cmd: git pull --ff-only tee: true - comment: Find version regexes - !Command - cmd: git checkout main && git pull --ff-only && git clean -df && git checkout -- . - comment: Go to main branch - logfile: checkout-main.log + cmd: git cherry-pick {{ persist_changelog_sha.changelog_sha }} && git push + comment: Cherry-pick changelog commit + logfile: cherry_pick_changelog_stable.log + tee: true - !Command - cmd: "{{ editor }} solr/CHANGES.txt" - comment: Edit CHANGES.txt for main branch, do necessary changes + cmd: git checkout main + comment: Checkout main branch stdout: true - !Command - cmd: git add -u . && git commit -m "Sync CHANGES for {{ release_version }}" && git push - logfile: commit-main.log - - !Command - cmd: git checkout {{ stable_branch }} && git pull --ff-only && git clean -df && git checkout -- . - comment: Go to stable branch - logfile: checkout-stable.log - - !Command - cmd: "{{ editor }} solr/CHANGES.txt" - comment: Edit CHANGES.txt for stable branch, do necessary changes - stdout: true + cmd: git pull --ff-only + tee: true - !Command - cmd: git add -u . && git commit -m "Sync CHANGES for {{ release_version }}" && git push - logfile: commit-stable.log + cmd: git cherry-pick {{ persist_changelog_sha.changelog_sha }} && git push + comment: Cherry-pick changelog commit + logfile: cherry_pick_changelog_main.log + tee: true - !Todo id: increment_release_version title: Add the next version on release branch @@ -1768,7 +1790,7 @@ groups: . Next to version {{ release_version_jira }}, click the gear pop-up menu icon and choose "Release" . Fill in the release date ({{ release_date | formatdate }}) - . It will give the option of transitioning issues marked fix-for the released version to the + . It will give the option of transitioning issues marked fix-for the released version to the next version, but do not do this as it will send an email for each issue :) links: - https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions @@ -1849,7 +1871,7 @@ groups: root_folder: '{{ git_checkout_folder }}' commands_text: | Run these commands to delete proposed versions from distribution directory. - + WARNING: Validate that the proposal is correct! commands: - !Command From 2e20ca0160789cfa49226079cae0bb225b007def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 08:52:38 +0200 Subject: [PATCH 43/64] License header for README Exclude rat for some changelog files --- changelog/README.md | 16 ++++++++++++++++ changelog/logchange-config.yml | 5 +++++ gradle/validation/rat-sources.gradle | 5 +++++ 3 files changed, 26 insertions(+) diff --git a/changelog/README.md b/changelog/README.md index 23a72bf74b13..86cde7c54492 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -1,3 +1,19 @@ + # New changelog process We are in the process of migrating to a new way of managing our changelog. Please see [dev-docs/changelog.md](../dev-docs/changelog.md) for details. diff --git a/changelog/logchange-config.yml b/changelog/logchange-config.yml index 849bee031099..c25458d09cb0 100644 --- a/changelog/logchange-config.yml +++ b/changelog/logchange-config.yml @@ -68,6 +68,11 @@ changelog: templates: entry: "${prefix}${title} ${merge_requests} ${issues} ${links} ${authors}" author: "([${name}](${url}) @${nick})" +# TODO: Using defaults for now, but want to test templates +# version_summary_templates: +# - path: version-summary.md +# changelog_templates: +# - path: CHANGELOG.md # see examples of templates at examples/templates # if you are missing some function, which will simplify your template (f.e getNumberOfEntries()) # feel free to create issue or pull request with change diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle index 37dde589a529..8f87dcb065db 100644 --- a/gradle/validation/rat-sources.gradle +++ b/gradle/validation/rat-sources.gradle @@ -96,6 +96,11 @@ allprojects { exclude "dev-tools/scripts/README.md" exclude "dev-tools/scripts/create_line_file_docs.py" + // Exclude new CHANGELOG and version-summary for each release + exclude "CHANGELOG.md" + exclude "changelog/**/version-summary.md" + exclude "changelog/.templates/*.md" + // The root project also includes patterns for the include composite // projects. Include their sources in the scan. include "build-tools/build-infra/src/**" From c5755afc573b35218616ce991ab0d188f0a5a8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 12:14:23 +0200 Subject: [PATCH 44/64] ExternalPaths check other path which is allowed Keep dockerfile-baseimage-java --- gradle/libs.versions.toml | 1 + .../src/java/org/apache/solr/util/ExternalPaths.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c278e2bfc087..3c9b0a9b6d2b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -79,6 +79,7 @@ cuvs-lucene = "25.10.0" cybozulabs-langdetect = "1.1-20120112" decompose = "3.3.0" diffplug-spotless = "7.2.1" +# @keep Use for dockerfile JRE version dockerfile-baseimage-java = "25" dropwizard-metrics = "4.2.26" eclipse-ecj = "3.39.0" diff --git a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java index f38c01bc3838..940ec58b9e94 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java +++ b/solr/test-framework/src/java/org/apache/solr/util/ExternalPaths.java @@ -19,6 +19,7 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import org.apache.solr.common.SolrException; /** * Some tests need to reach outside the classpath to get certain resources (e.g. the example @@ -82,13 +83,14 @@ static Path determineSourceHome() { } Path base = file.toAbsolutePath(); - while (!Files.exists(base.resolve("CHANGELOG.md")) && null != base) { + while (!Files.exists(base.resolve("solr/test-framework/build.gradle")) && null != base) { base = base.getParent(); } return (null == base) ? null : base.resolve("solr/").toAbsolutePath(); } catch (Exception e) { // all bets are off - return null; + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Failed to determine source home", e); } } } From b1db3ff7e4ad0ccd9103eb5dab21a888b775bc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 12:23:12 +0200 Subject: [PATCH 45/64] Clarify docs --- dev-docs/changelog.adoc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 48bc96690710..f47e10c53525 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -82,7 +82,7 @@ The `validate-changelog` GitHub workflow automatically checks that: . **CHANGES.txt is not edited directly** - All changes must use the YAML fragment approach . **A changelog entry is added** - Code changes must include a corresponding YAML file in `changelog/unreleased/` -If your PR modifies code but should not have a changelog entry (e.g., internal refactoring, build system changes), add the `no-changelog` label to skip these checks. +If your change does not require a changelog entry, it is still possible to merge the PR. == 5. Writing Good Entries @@ -118,10 +118,11 @@ links: == 7. Migration tool -There is a migration tool in `dev-tools/scripts/changes2logchange.py`. +There is a migration tool in `dev-tools/scripts/changes2logchange.py` for one-time use during the transition. It will bulk convert the entire `solr/CHANGES.txt` file to files in the `changelog/` folder and versioned sub folders. -The tool can also convert a single changelog entry reading `stdin` as follows: +The tool can also be used by a developer who needs to convert many entries from work-in-progress after the migration +cutover. You can paste one or more entries and get them converted in the Terminal as follows: ```bash cat << EOF | python3 dev-tools/scripts/changes2logchange.py - From b130f5325474989a039af65caaf2291875a8bdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 13:30:42 +0200 Subject: [PATCH 46/64] Fix releaseWizard.yaml syntax errors --- .editorconfig | 3 +++ dev-tools/scripts/releaseWizard.yaml | 16 +++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index e127b9a9e5db..a98ddb5476ed 100644 --- a/.editorconfig +++ b/.editorconfig @@ -671,6 +671,9 @@ ij_yaml_space_before_colon = false ij_yaml_spaces_within_braces = true ij_yaml_spaces_within_brackets = true +[dev-tools/scripts/releaseWizard.yaml] +trim_trailing_whitespace = false + # dos files [{*.bat,*.cmd}] end_of_line = crlf \ No newline at end of file diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 6b36f74010e1..4c16a1f6557a 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -77,7 +77,7 @@ templates: Reason for fail is {{ reason }}. {% endif %} - This vote has {% if passed %}PASSED{% else %}FAILED{% endif %} + This vote has {% if passed %}PASSED{% else %}FAILED{% endif %} ---- {%- endmacro %} announce_solr: | @@ -665,7 +665,7 @@ groups: - bugfix - !Todo id: dependency_updates_changes - title: Add dependency updates to CHANGELOG.md + title: Add dependency updates to changelog description: Bulk add all 'solrbot' dependency updates since last release depends: clean_git_checkout commands: !Commands @@ -686,7 +686,8 @@ groups: id: generate_changelog_release title: Generate CHANGELOG.md for release description: | - Generate `CHANGELOG.md` by running `logchange release` and `logchange generate`. This will prepare both a `changelog/v{{ release_version }}` folder and the `CHANGELOG.md` file in one commit. + Generate `CHANGELOG.md` by running `logchange release` and `logchange generate`. This will prepare both a + `changelog/v{{ release_version }}` folder and the `CHANGELOG.md` file in one commit. depends: dependency_updates_changes commands: !Commands root_folder: '{{ git_checkout_folder }}' @@ -707,7 +708,7 @@ groups: comment: Generate `CHANGELOG.md` in repository root tee: true - !Command - cmd: git add CHANGELOG.md changelog/v* && git commit -m "Changelog for release v{{ release_version }}" && git push + cmd: git add CHANGELOG.md changelog && git commit -m "Changelog for release v{{ release_version }}" && git push comment: Commit and push changelog changes logfile: commit_changelog.log tee: true @@ -717,8 +718,8 @@ groups: description: Store the current git sha depends: generate_changelog_release vars: - changelog_sha: {{ current_git_rev }} - persist: changelog_sha + changelog_sha: '{{ current_git_rev }}' + persist_vars: changelog_sha - !Todo id: draft_release_notes title: Get a draft of the release notes in place @@ -726,7 +727,8 @@ groups: These are typically edited on the Wiki Clone a page for a previous version as a starting point for your release notes. - Edit the contents of `CHANGELOG.md` into a more concise format for public consumption. + Use the contents of `CHANGELOG.md` and `major-changes-in-solr-N.adoc` as input and write a more + concise format for public consumption. Ask on dev@ for input. Ideally the timing of this request mostly coincides with the release branch creation. It's a good idea to remind the devs of this later in the release too. From 3fc4e84d0e131f277c96f00274974daba482c68b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 14:48:53 +0200 Subject: [PATCH 47/64] Correct spelling of dev-docs/changelog.adoc --- CHANGELOG.md | 2 +- changelog/README.md | 2 +- gradle/template.gradle.properties | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d34ba1a95b6..cb8dfd9fbc14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ DUMMY changelog file. -We are in the process of migrating from CHANGES.txt to a structured approach to changelog generation. See [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) as well as [dev-docs/changelog.md](dev-docs/changelog.md) for details. +We are in the process of migrating from CHANGES.txt to a structured approach to changelog generation. See [SOLR-17619](https://issues.apache.org/jira/browse/SOLR-17619) as well as [dev-docs/changelog.adoc](dev-docs/changelog.adoc) for details. [unreleased] ------------ diff --git a/changelog/README.md b/changelog/README.md index 86cde7c54492..9947e6006284 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -16,6 +16,6 @@ --> # New changelog process -We are in the process of migrating to a new way of managing our changelog. Please see [dev-docs/changelog.md](../dev-docs/changelog.md) for details. +We are in the process of migrating to a new way of managing our changelog. Please see [dev-docs/changelog.adoc](../dev-docs/changelog.adoc) for details. In a transition period it is still possible to merge your changelog entry to `solr/CHANGES.txt`, but on Wednesday October 29th this will not longer be the case. diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index 7e710744c4df..09a4856e9eca 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -125,7 +125,7 @@ production=false # parameter. disableUiModule=false -# Changelog generator settings (./gradlew generateChangeYaml, see dev-docs/changelog.md) +# Changelog generator settings (./gradlew generateChangeYaml, see dev-docs/changelog.adoc) #user.name= #user.githubid= -#user.asfid= \ No newline at end of file +#user.asfid= From 2c8f6dbf6a86c392f8c40590ecabac3dc6382c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:09:08 +0200 Subject: [PATCH 48/64] Rename task Use dev-docs link in comment --- .github/workflows/validate-changelog.yml | 4 ++-- dev-docs/changelog.adoc | 8 ++++---- gradle/changelog.gradle | 4 ++-- gradle/template.gradle.properties | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index c518ccb3dcde..b282b57ba461 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -36,7 +36,7 @@ jobs: echo "::error::Use of solr/CHANGES.txt is deprecated. Please create a changelog yaml file instead." echo "" echo "Instead of editing CHANGES.txt, please:" - echo "1. Run: ./gradlew generateChangeYaml" + echo "1. Run: ./gradlew changelog" echo "2. Edit the generated YAML file in changelog/unreleased/" echo "3. Commit both the code change and the YAML file" echo "" @@ -77,7 +77,7 @@ jobs: echo "::error::This PR appears to contain code changes but no changelog entry was added." echo "" echo "Please add a changelog entry by:" - echo "1. Running: ./gradlew generateChangeYaml" + echo "1. Running: ./gradlew changelog" echo "2. Editing the generated YAML file in changelog/unreleased/" echo "3. Committing the YAML file" echo "" diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index f47e10c53525..f7c1108bfef8 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -15,7 +15,7 @@ available, and finishes with a concrete example fragment. . Make a feature branch with a descriptive name like 'SOLR-12345-fix-memory-leak'. . Implement your change -. Run `./gradlew generateChangeYaml` +. Run `./gradlew changelog` . Edit the generated YAML file in `changelog/unreleased/` to add a clear, user-focused description. . Commit the YAML fragment along with the code change. . Open your PR as usual. @@ -34,7 +34,7 @@ solr/ == 3. Gradle Tasks -=== 3.1 Our Custom Task: `generateChangeYaml` +=== 3.1 Our Custom Task: `changelog` Bootstraps a YAML file, by default in the `changelog/unreleased/` directory for the current branch, The task will use branch name as file name and title, and will @@ -47,7 +47,7 @@ use that information. Example: [source, properties] ---- -# Changelog generator settings (./gradlew generateChangeYaml) +# Changelog generator settings (./gradlew changelog) user.name=John Doe user.githubid=johngithub user.asfid=johnapache @@ -57,7 +57,7 @@ Invoke the task with: [source, bash] ---- -./gradlew generateChangeYaml +./gradlew changelog ---- === 3.2 Plugin Tasks diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 1f7d915193ff..6d4e925e4149 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -22,7 +22,7 @@ logchange { generateChangesXml = false } -task generateChangeYaml { +task changelog { description = 'Generates a change/log description file (YAML)' doLast { def gitUserName = 'git config user.name'.execute().text.trim() @@ -43,7 +43,7 @@ task generateChangeYaml { def fileName = "changelog/unreleased/${gitBranch}.yml" def file = new File(fileName) file.parentFile.mkdirs() - file.text = """# logchange format: https://github.com/logchange/logchange#yaml-format + file.text = """# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc title: ${branchWithoutJira} type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other authors: diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index 09a4856e9eca..65dbf7d01452 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -125,7 +125,7 @@ production=false # parameter. disableUiModule=false -# Changelog generator settings (./gradlew generateChangeYaml, see dev-docs/changelog.adoc) +# Changelog generator settings (./gradlew changelog, see dev-docs/changelog.adoc) #user.name= #user.githubid= #user.asfid= From 9dc20a07258b17c2219095997a28f5b2be136bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:19:02 +0200 Subject: [PATCH 49/64] Clean up the changelog.adoc doc --- dev-docs/changelog.adoc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index f7c1108bfef8..8c6813628cd2 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -62,17 +62,15 @@ Invoke the task with: === 3.2 Plugin Tasks -[cols="1,2,3", options="header"] +[cols="1,2", options="header"] |=== -| Task | Purpose | Typical Usage +| Task | Purpose | `logchangeGenerate` -| Generates changelog file () based on .yml entries and archives (does not moves any files) -| Doc generation, CI/CD +| Generates changelog file `/CHANGELOG.md` based on `.yml` entries and archives (does not moves any files) | `logchangeRelease` -| Creates new changelog release by moving files from directory to vX.Y.Z directory -| Run by the Release Manager during the release procedure. +| Creates a new changelog release by moving files from `changelog/unreleased/` directory to `changelog/vX.Y.Z` directory |=== == 4. Changelog Validation in Pull Requests @@ -87,6 +85,7 @@ If your change does not require a changelog entry, it is still possible to merge == 5. Writing Good Entries * **Audience** is end-users and administrators, not committers. +* If the change is super minor, like a typo, don't bother adding a yaml file * Keep the entry short and focused on the user impact. * Choose the correct *type*: ** `added` for new features @@ -98,14 +97,13 @@ If your change does not require a changelog entry, it is still possible to merge ** `security` for security-related changes ** `other` for anything else, like the build or documentation * Reference issues as `SOLR-12345` or GitHub `PR#123`. -* If the change is super minor, like a typo, don't bother adding a yaml file == 6. Example YAML Fragment [source, yaml] ---- title: Fix nasty bug in CoreContainer shutdown -# added, changed, optimized, fixed, deprecated, removed, dependency_update, security, other +# added, changed, fixed, deprecated, removed, dependency_update, security, other type: fixed authors: - name: John Contributor @@ -124,26 +122,28 @@ It will bulk convert the entire `solr/CHANGES.txt` file to files in the `changel The tool can also be used by a developer who needs to convert many entries from work-in-progress after the migration cutover. You can paste one or more entries and get them converted in the Terminal as follows: -```bash +[source,bash] +---- cat << EOF | python3 dev-tools/scripts/changes2logchange.py - * SOLR-17960: Removed TikaLanguageIdentifierUpdateProcessor. Use LangDetectLanguageIdentifierUpdateProcessor or OpenNLPLangDetectUpdateProcessor instead. (janhoy) EOF -``` +---- which will output to `stdout`: -``` +[source,yaml] +---- title: Removed TikaLanguageIdentifierUpdateProcessor. Use LangDetectLanguageIdentifierUpdateProcessor or OpenNLPLangDetectUpdateProcessor instead. -type: other +type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other authors: - name: janhoy links: - name: SOLR-17960 url: https://issues.apache.org/jira/browse/SOLR-17960 -``` +---- == 8. Further Reading From adc0b09d6120e020bb596a963079e38b82ab2536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:33:34 +0200 Subject: [PATCH 50/64] Re-organize adoc --- dev-docs/changelog.adoc | 119 +++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 8c6813628cd2..12f31328fda4 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -5,11 +5,13 @@ The project no longer maintains a monolithic *CHANGES.txt* file directly. Instead, we create small YAML fragments **per bug-fix or feature**, checked into -`solr/changelog/v1.2.3/`. At release time these fragments are automatically -collated into a human-readable *CHANGES.md*. +the `solr/changelog/unreleased/ folder`. At release time these fragments are +organized in versioned folders and collated into a human-readable *CHANGELOG.md* file. This document walks you through the workflow, explains the Gradle helpers -available, and finishes with a concrete example fragment. +available, and finishes with a concrete example fragment. We use a tool called +[logchange](https://logchange.dev/tools/logchange/) as a gradle plugin to manage +the changelog. == 1. Workflow Overview @@ -32,15 +34,37 @@ solr/ └── v10.0.1/ ---- -== 3. Gradle Tasks +== 3. The YAML format -=== 3.1 Our Custom Task: `changelog` +Below is an example of a changelog yaml fragment. The full yaml format is xref:https://logchange.dev/tools/logchange/reference/#tasks[documented here], but we normally only need `title`, `type`, `authors` and `links`: -Bootstraps a YAML file, by default in the `changelog/unreleased/` directory for -the current branch, The task will use branch name as file name and title, and will -parse JIRA id from the branch name if it exists. +[source, yaml] +---- +title: Fix nasty bug in CoreContainer shutdown +# added, changed, fixed, deprecated, removed, dependency_update, security, other +type: fixed +authors: + - name: John Contributor + nick: johngithub + url: https://home.apache.org/phonebook.html?uid=johnasfid +links: + - name: SOLR-3333 + url: https://issues.apache.org/jira/browse/SOLR-3333 +---- + +=== 3.1 Tool to draft a YAML for your change + +We have a gradle task that bootstraps a YAML file in the `changelog/unreleased/` directory. The task will use your current branch name as a file name and also title, and will +try to parse JIRA id from the branch name if it exists to add the `links`. + +Invoke the task with: + +[source, bash] +---- +./gradlew changelog +---- -It will use your git `user.name` as display name by default, but if you +The task will use your git `user.name` as display name by default, but if you add your name, GitHub username, and optionally Apache ID to the `gradle.properties` file at the root of the Solr git checkout, this task will use that information. Example: @@ -53,14 +77,36 @@ user.githubid=johngithub user.asfid=johnapache ---- -Invoke the task with: +== 4. Writing Good Entries -[source, bash] ----- -./gradlew changelog ----- +* **Audience** is end-users and administrators, not committers. +* If the change is super minor, like a typo, don't bother adding a yaml file +* Keep the entry short and focused on the user impact. +* Choose the correct *type*: +** `added` for new features +** `changed` for improvements to existing code +** `fixed` for bug fixes +** `deprecated` for deprecated features +** `removed` for code removed in major releases +** `dependency_update` for updates to dependencies +** `security` for security-related changes +** `other` for anything else, like the build or documentation +* Reference issues as `SOLR-12345` or GitHub `PR#123`. -=== 3.2 Plugin Tasks +== 5. Changelog Validation in Pull Requests + +The `validate-changelog` GitHub workflow automatically checks that: + +. **CHANGES.txt is not edited directly** - All changes must use the YAML fragment approach +. **A changelog entry is added** - Code changes must include a corresponding YAML file in `changelog/unreleased/` + +If your change does not require a changelog entry, it is still possible to merge the PR. + +== 6. For release managers + +=== 6.1 Gradle tasks for logchange + +The logchange gradle plugin offers some tasks, here are the two most important: [cols="1,2", options="header"] |=== @@ -73,48 +119,9 @@ Invoke the task with: | Creates a new changelog release by moving files from `changelog/unreleased/` directory to `changelog/vX.Y.Z` directory |=== -== 4. Changelog Validation in Pull Requests - -The `validate-changelog` GitHub workflow automatically checks that: - -. **CHANGES.txt is not edited directly** - All changes must use the YAML fragment approach -. **A changelog entry is added** - Code changes must include a corresponding YAML file in `changelog/unreleased/` - -If your change does not require a changelog entry, it is still possible to merge the PR. - -== 5. Writing Good Entries - -* **Audience** is end-users and administrators, not committers. -* If the change is super minor, like a typo, don't bother adding a yaml file -* Keep the entry short and focused on the user impact. -* Choose the correct *type*: - ** `added` for new features - ** `changed` for improvements to existing code - ** `fixed` for bug fixes - ** `deprecated` for deprecated features - ** `removed` for code removed in major releases - ** `dependency_update` for updates to dependencies - ** `security` for security-related changes - ** `other` for anything else, like the build or documentation -* Reference issues as `SOLR-12345` or GitHub `PR#123`. - -== 6. Example YAML Fragment - -[source, yaml] ----- -title: Fix nasty bug in CoreContainer shutdown -# added, changed, fixed, deprecated, removed, dependency_update, security, other -type: fixed -authors: - - name: John Contributor - nick: johngithub - url: https://home.apache.org/phonebook.html?uid=johnasfid -links: - - name: SOLR-3333 - url: https://issues.apache.org/jira/browse/SOLR-3333 ----- +These are integrated in the Release Wizard. -== 7. Migration tool +=== 6.2 Migration tool There is a migration tool in `dev-tools/scripts/changes2logchange.py` for one-time use during the transition. It will bulk convert the entire `solr/CHANGES.txt` file to files in the `changelog/` folder and versioned sub folders. From a4f661f76ae42673a6fead023a17a5cd9d5cd68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:42:17 +0200 Subject: [PATCH 51/64] Rename task once again as writeChangelog with an alias changelog --- .github/workflows/validate-changelog.yml | 4 ++-- dev-docs/changelog.adoc | 6 +++--- gradle/changelog.gradle | 12 +++++++++++- gradle/template.gradle.properties | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index b282b57ba461..24582428c9fe 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -36,7 +36,7 @@ jobs: echo "::error::Use of solr/CHANGES.txt is deprecated. Please create a changelog yaml file instead." echo "" echo "Instead of editing CHANGES.txt, please:" - echo "1. Run: ./gradlew changelog" + echo "1. Run: ./gradlew writeChangelog" echo "2. Edit the generated YAML file in changelog/unreleased/" echo "3. Commit both the code change and the YAML file" echo "" @@ -77,7 +77,7 @@ jobs: echo "::error::This PR appears to contain code changes but no changelog entry was added." echo "" echo "Please add a changelog entry by:" - echo "1. Running: ./gradlew changelog" + echo "1. Running: ./gradlew writeChangelog" echo "2. Editing the generated YAML file in changelog/unreleased/" echo "3. Committing the YAML file" echo "" diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 12f31328fda4..43a76bd347f3 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -17,7 +17,7 @@ the changelog. . Make a feature branch with a descriptive name like 'SOLR-12345-fix-memory-leak'. . Implement your change -. Run `./gradlew changelog` +. Run `./gradlew writeChangelog` . Edit the generated YAML file in `changelog/unreleased/` to add a clear, user-focused description. . Commit the YAML fragment along with the code change. . Open your PR as usual. @@ -61,7 +61,7 @@ Invoke the task with: [source, bash] ---- -./gradlew changelog +./gradlew writeChangelog ---- The task will use your git `user.name` as display name by default, but if you @@ -71,7 +71,7 @@ use that information. Example: [source, properties] ---- -# Changelog generator settings (./gradlew changelog) +# Changelog generator settings (./gradlew writeChangelog) user.name=John Doe user.githubid=johngithub user.asfid=johnapache diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle index 6d4e925e4149..cd54cce9ad77 100644 --- a/gradle/changelog.gradle +++ b/gradle/changelog.gradle @@ -22,7 +22,7 @@ logchange { generateChangesXml = false } -task changelog { +task writeChangelog { description = 'Generates a change/log description file (YAML)' doLast { def gitUserName = 'git config user.name'.execute().text.trim() @@ -55,3 +55,13 @@ ${githubLink} println "Generated file: ${fileName} -- open it" } } + +task changelog { + dependsOn writeChangelog + description = 'Generates a change/log description file (YAML)' +} + +task newChangelog { + dependsOn writeChangelog + description = 'Generates a change/log description file (YAML)' +} diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index 65dbf7d01452..b6ae69b85e82 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -125,7 +125,7 @@ production=false # parameter. disableUiModule=false -# Changelog generator settings (./gradlew changelog, see dev-docs/changelog.adoc) +# Changelog generator settings (./gradlew writeChangelog, see dev-docs/changelog.adoc) #user.name= #user.githubid= #user.asfid= From 59dd6610939dfb000c1e5012498be6e474d533e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:45:26 +0200 Subject: [PATCH 52/64] Wrong numbering in adoc --- dev-docs/changelog.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index 43a76bd347f3..b553a6598b98 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -152,7 +152,7 @@ links: url: https://issues.apache.org/jira/browse/SOLR-17960 ---- -== 8. Further Reading +== 7. Further Reading * xref:https://github.com/logchange/logchange[Logchange web page] * xref:https://keepachangelog.com/en/1.1.0/[keepachangelog.com website] From 9cc30b04641da00c1d66fbbb0e328d2521fb1310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:56:38 +0200 Subject: [PATCH 53/64] Improve addDepsTochanges - Nicer filenames - Extract jira issues from title / commit msg --- dev-tools/scripts/addDepsToChanges.py | 24 ++++++++++++-------- dev-tools/scripts/scriptutil.py | 32 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py index a53066b0a757..8fecc5573d28 100755 --- a/dev-tools/scripts/addDepsToChanges.py +++ b/dev-tools/scripts/addDepsToChanges.py @@ -62,22 +62,28 @@ def __str__(self) -> str: def to_yaml_dict(self) -> dict: """ - Convert to a dictionary suitable for YAML serialization + Convert to a dictionary suitable for YAML serialization. + Extracts JIRA IDs from the title and adds them to links. """ + # Extract JIRA IDs from the message + title, jira_links = extract_jira_issues_from_title(self.message) + + # Build links: JIRA issues first, then PR + links = jira_links.copy() # Start with JIRA links + links.append({ + 'name': f'PR#{self.pr_num}', + 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' + }) + return { - 'title': self.message, + 'title': title, 'type': 'dependency_update', 'authors': [ { 'name': self.author } ], - 'links': [ - { - 'name': f'PR#{self.pr_num}', - 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' - } - ] + 'links': links } def yaml_filename(self) -> str: @@ -90,7 +96,7 @@ def yaml_filename(self) -> str: # Replace spaces with dashes slug = re.sub(r'\s+', '-', slug) # Remove non-alphanumeric except dashes - slug = re.sub(r'[^a-z0-9-]', '', slug) + slug = re.sub(r'[^a-z0-9-._]', '', slug) # Truncate to reasonable length slug = slug[:50] # Remove trailing dashes diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py index c766077356f0..b87c22b3e83b 100644 --- a/dev-tools/scripts/scriptutil.py +++ b/dev-tools/scripts/scriptutil.py @@ -244,6 +244,38 @@ def find_current_lucene_version(): return lucene_version_prop_re.search(versions_file).group(1).strip() +def extract_jira_issues_from_title(title): + """ + Extract JIRA issue IDs from a title/text string. + Returns a tuple of (cleaned_title, list_of_jira_links) + where jira_links is a list of dicts with 'name' and 'url' keys. + + Handles patterns like: SOLR-12345, LUCENE-1234, INFRA-567 + Only extracts JIRA issues, not GitHub PRs. + """ + jira_pattern = re.compile(r'(?:SOLR|LUCENE|INFRA)-(\d+)') + + links = [] + seen_issues = set() + + # Find all JIRA issues in the title + for match in jira_pattern.finditer(title): + issue_id = match.group(0) # Full "SOLR-12345" format + if issue_id not in seen_issues: + url = f"https://issues.apache.org/jira/browse/{issue_id}" + links.append({ + 'name': issue_id, + 'url': url + }) + seen_issues.add(issue_id) + + # Remove JIRA IDs from the title (only at the beginning, followed by colon or space) + # Pattern: "SOLR-12345: " or "SOLR-12345 " at the start + cleaned_title = re.sub(r'^(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', '', title).strip() + + return cleaned_title, links + + if __name__ == '__main__': print('This is only a support module, it cannot be run') sys.exit(1) From e4a89333cc4673d144b96b8c1567de23731a31d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 15:56:38 +0200 Subject: [PATCH 54/64] Improve addDepsTochanges - Nicer filenames - Extract jira issues from title / commit msg --- dev-tools/scripts/addDepsToChanges.py | 51 +++++++++++++++++++------- dev-tools/scripts/scriptutil.py | 53 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py index a53066b0a757..96a5c2ec671d 100755 --- a/dev-tools/scripts/addDepsToChanges.py +++ b/dev-tools/scripts/addDepsToChanges.py @@ -62,39 +62,64 @@ def __str__(self) -> str: def to_yaml_dict(self) -> dict: """ - Convert to a dictionary suitable for YAML serialization + Convert to a dictionary suitable for YAML serialization. + Extracts JIRA IDs from the title and adds them to links. """ + # Extract JIRA IDs from the message + title, jira_links = extract_jira_issues_from_title(self.message) + + # Build links: JIRA issues first, then PR + links = jira_links.copy() # Start with JIRA links + links.append({ + 'name': f'PR#{self.pr_num}', + 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' + }) + return { - 'title': self.message, + 'title': title, 'type': 'dependency_update', 'authors': [ { 'name': self.author } ], - 'links': [ - { - 'name': f'PR#{self.pr_num}', - 'url': f'https://github.com/apache/solr/pull/{self.pr_num}' - } - ] + 'links': links } def yaml_filename(self) -> str: """ Generate a filesystem-safe filename for this entry. Format: PR#####-slug.yaml + Truncates slug on whitespace boundaries, allowing up to 255 chars total. """ # Clean message for slug slug = self.message.lower() # Replace spaces with dashes slug = re.sub(r'\s+', '-', slug) # Remove non-alphanumeric except dashes - slug = re.sub(r'[^a-z0-9-]', '', slug) - # Truncate to reasonable length - slug = slug[:50] - # Remove trailing dashes - slug = slug.rstrip('-') + slug = re.sub(r'[^a-z0-9-._]', '', slug) + + # Calculate available space for slug + # Format: "PR" + pr_num + "-" + slug + ".yaml" + # Typical PR#1234 = 8 chars + "-" = 9 chars, ".yaml" = 5 chars, total overhead = 14 chars + # Most filesystems limit filenames to 255 chars + max_filename_length = 255 + overhead = len(f"PR{self.pr_num}-.yaml") + max_slug_length = max_filename_length - overhead + + # Truncate to max length on word boundaries if necessary + if len(slug) > max_slug_length: + # Find the last dash within the limit + truncated = slug[:max_slug_length] + last_dash = truncated.rfind('-') + if last_dash > max_slug_length // 2: # Keep at least half the available space + slug = truncated[:last_dash] + else: + # If no good word boundary, use hard limit and clean up trailing dashes + slug = truncated.rstrip('-') + else: + # Remove trailing dashes + slug = slug.rstrip('-') return f"PR{self.pr_num}-{slug}.yaml" diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py index c766077356f0..93cb426cbe5f 100644 --- a/dev-tools/scripts/scriptutil.py +++ b/dev-tools/scripts/scriptutil.py @@ -244,6 +244,59 @@ def find_current_lucene_version(): return lucene_version_prop_re.search(versions_file).group(1).strip() +def extract_jira_issues_from_title(title): + """ + Extract JIRA issue IDs from a title/text string. + Returns a tuple of (cleaned_title, list_of_jira_links) + where jira_links is a list of dicts with 'name' and 'url' keys. + + Handles patterns like: SOLR-12345, LUCENE-1234, INFRA-567 + Only extracts JIRA issues, not GitHub PRs. + Handles both "SOLR-123: description" and "description / SOLR-456:" patterns. + """ + jira_pattern = re.compile(r'(?:SOLR|LUCENE|INFRA)-(\d+)') + + links = [] + seen_issues = set() + + # Find all JIRA issues in the title + for match in jira_pattern.finditer(title): + issue_id = match.group(0) # Full "SOLR-12345" format + if issue_id not in seen_issues: + url = f"https://issues.apache.org/jira/browse/{issue_id}" + links.append({ + 'name': issue_id, + 'url': url + }) + seen_issues.add(issue_id) + + # Remove JIRA IDs from the title. + # Handle multiple patterns: + # 1. "SOLR-12345: description" at the beginning + # 2. "description / SOLR-456: more text" (remove the slash and JIRA ID) + # 3. "/ SOLR-123: description" (leading slash with JIRA) + # 4. Multiple JIRA IDs anywhere in the title + + cleaned_title = title + + # First, clean up leading slash patterns like "/ SOLR-123:" or "/ SOLR-123 " + cleaned_title = re.sub(r'^\s*/\s*(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', '', cleaned_title) + + # Remove slash-separated JIRA IDs in the middle: "... / SOLR-123:" or "... / SOLR-123 " + cleaned_title = re.sub(r'\s+/\s*(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', ' ', cleaned_title) + + # Remove any remaining JIRA IDs at the beginning followed by colon/space + cleaned_title = re.sub(r'^(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', '', cleaned_title) + + # Clean up any remaining leading slashes or extra whitespace + cleaned_title = re.sub(r'^\s*/\s*', '', cleaned_title).strip() + + # Collapse multiple spaces + cleaned_title = re.sub(r'\s+', ' ', cleaned_title) + + return cleaned_title, links + + if __name__ == '__main__': print('This is only a support module, it cannot be run') sys.exit(1) From e874d67da9f9d6333faeaf5724c8fafc42ade4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 16:28:46 +0200 Subject: [PATCH 55/64] Revert unneccessary whitespace change in wiz --- dev-tools/scripts/releaseWizard.yaml | 42 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index 4c16a1f6557a..a5493e6ddead 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -58,7 +58,7 @@ templates: {% set passed = plus_binding >= 3 and minus < plus_binding %} {% set too_few = plus_binding < 3 %} {% set veto = plus_binding < minus %} - {% set reason = 'too few binding votes' if too_few else 'too many negative votes' if veto else 'unknown' %} + {% set reason = 'too few binding votes' if too_few else 'too many negative votes' if veto else 'unknown' %} vote_macro: | {% macro end_vote_result(plus_binding, plus_other, zero, minus) -%} (( template=vote_logic )) @@ -75,7 +75,7 @@ templates: {% if not passed %} Reason for fail is {{ reason }}. - + {% endif %} This vote has {% if passed %}PASSED{% else %}FAILED{% endif %} ---- @@ -116,10 +116,10 @@ templates: (( template=announce_solr_mail_body )) ---- - announce_solr_announce_mail: | + announce_solr_announce_mail: | The template below can be used to announce the Solr release to the `announce@apache.org` mailing list. The email *must* be sent *from your `@apache.org` account*, - and it *can optionally* be signed with your PGP key, using the `--clearsign` option of + and it *can optionally* be signed with your PGP key, using the `--clearsign` option of gpg, as explained in https://www.gnupg.org/gph/en/manual/x135.html. .Mail template @@ -166,12 +166,12 @@ templates: # # Vars can contain global jinja vars or local vars earlier defined (ordered dict) # persist_vars: ['var_name', 'var_name'] # List of variables to persist in TODO state # asciidoc: | -# Some `asciidoc` text to be included in asciidoc guide +# Some `asciidoc` text to be included in asciidoc guide # *instead of* description/post_description # function: my_python_function # Will call the named function for complex tasks # commands: !Commands # A !Commands object holding commands to execute for this todo # root_folder: '{{ git_checkout_folder }}' # path to where commands will run -# commands_text: Introduction text to be displayed just before the commands +# commands_text: Introduction text to be displayed just before the commands # enable_execute: true # Set to false to never offer to run commands automatically # confirm_each_command: true # Set to false to run all commands without prompting # remove_files: ['file1', 'folder2'] # List of files or folders that must be gone @@ -196,7 +196,7 @@ templates: # name: gpg_id # This will be stored in todo state and can be referenced as {{ todo_id.name }} # type: int # if no type is given, a string is stored. Supported types are 'int' # post_description: | -# Some `asciidoc` text (with jinja template support) +# Some `asciidoc` text (with jinja template support) # to be printed *after* commands and user_input is done. # links: # - http://example.com/list/of/links?to&be&displayed @@ -453,7 +453,7 @@ groups: title: Update Ref Guide Metadata for new Minor Branch description: | This will update the "solr/solr-ref-guide/antora.yml" file. - + The file shouldn't change much, but it should pick up new dependency versions, if they have changed. types: - major @@ -557,7 +557,7 @@ groups: description: | This will make sure we are continuously testing the new branch(es) we have created. Consult the JenkinsReleaseBuilds page for more information. - + Job Suffixes to create: {% if release_type == 'major' %} *-{{release_version_major}}.x (The new stable branch) @@ -575,7 +575,7 @@ groups: description: | This will make sure we aren't using resources to test branches no longer in development. Consult the JenkinsReleaseBuilds page for more information. - + Job Suffixes to delete: {% if release_type == 'major' %} *-{{latest_lts_version_major}}.{{latest_lts_version_minor}} (The previous LTS release branch) @@ -832,7 +832,7 @@ groups: - !Todo id: build_rc title: Build the release candidate - depends: + depends: - setup_gpg - run_tests vars: @@ -853,7 +853,7 @@ groups: Note that the build will take a long time. To follow the detailed build log, you can tail the log file {{ logfile | default("") }}. confirm_each_command: false - remove_files: + remove_files: - '{{ dist_file_path }}' commands: - !Command @@ -912,7 +912,7 @@ groups: id: verify_staged title: Verify staged artifacts description: | - A lightweight smoke testing which downloads the artifacts from stage + A lightweight smoke testing which downloads the artifacts from stage area and checks hash and signatures, but does not re-run all tests. depends: import_svn vars: @@ -1056,7 +1056,7 @@ groups: there are more positive than negative votes you can techically release the software. However, please review the negative votes and consider a re-spin. - + {% endif %} {%- endif %} {{ end_vote_result(plus_binding,plus_other,zero,minus) }} @@ -1128,10 +1128,10 @@ groups: depends: mv_to_release description: | For every release, we publish the official dockerfile for solr via the methods required by Docker. - + This will start the process, and will ultimately create a PR in https://github.com/docker-library/official-images. This PR creation is automated, you do not need to do more work beyond these steps. - + When that PR is approved and merged, the Official image for this Solr version will be built and available for use. vars: docker_version: '{{ release_version_major }}.{{ release_version_minor }}' @@ -1275,7 +1275,7 @@ groups: The Solr minor branches do not contain up-to-date ref-guide metadata (antora.yml). Instead, they have the information from the last release. This allows us to build the official documentation from these minor branches, instead of from the release tags. - + This means that we can make fixes to older ref-guides without requiring a new release for those versions. depends: tag_release commands: !Commands @@ -1555,7 +1555,7 @@ groups: title: Update the DOAP file description: | Update the Solr DOAP RDF file on the unstable, stable and release branches to - reflect the new versions (note that the website .htaccess file redirects from their + reflect the new versions (note that the website .htaccess file redirects from their canonical URLs to their locations in the Solr Git source repository - see dev-tools/doap/README.txt for more info) commands: !Commands @@ -1619,7 +1619,7 @@ groups: id: announce title: Announce the release description: | - For feature releases, your announcement should describe the main features included + For feature releases, your announcement should describe the main features included in the release. *Send the announce as Plain-text email, not HTML.* This step will generate email templates based on the news files you edited earlier for the website. @@ -1792,7 +1792,7 @@ groups: . Next to version {{ release_version_jira }}, click the gear pop-up menu icon and choose "Release" . Fill in the release date ({{ release_date | formatdate }}) - . It will give the option of transitioning issues marked fix-for the released version to the + . It will give the option of transitioning issues marked fix-for the released version to the next version, but do not do this as it will send an email for each issue :) links: - https://issues.apache.org/jira/plugins/servlet/project-config/SOLR/versions @@ -1873,7 +1873,7 @@ groups: root_folder: '{{ git_checkout_folder }}' commands_text: | Run these commands to delete proposed versions from distribution directory. - + WARNING: Validate that the proposal is correct! commands: - !Command From 0fe8fc7687969c0b9089de457d5d8d43350166bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 16:33:30 +0200 Subject: [PATCH 56/64] Rewrite extract_jira_issues_from_title --- dev-tools/scripts/scriptutil.py | 69 +++++++++------------------------ 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/dev-tools/scripts/scriptutil.py b/dev-tools/scripts/scriptutil.py index 4c0bd907ae95..2318b7159f86 100644 --- a/dev-tools/scripts/scriptutil.py +++ b/dev-tools/scripts/scriptutil.py @@ -245,56 +245,25 @@ def find_current_lucene_version(): def extract_jira_issues_from_title(title): - """ - Extract JIRA issue IDs from a title/text string. - Returns a tuple of (cleaned_title, list_of_jira_links) - where jira_links is a list of dicts with 'name' and 'url' keys. - - Handles patterns like: SOLR-12345, LUCENE-1234, INFRA-567 - Only extracts JIRA issues, not GitHub PRs. - Handles both "SOLR-123: description" and "description / SOLR-456:" patterns. - """ - jira_pattern = re.compile(r'(?:SOLR|LUCENE|INFRA)-(\d+)') - - links = [] - seen_issues = set() - - # Find all JIRA issues in the title - for match in jira_pattern.finditer(title): - issue_id = match.group(0) # Full "SOLR-12345" format - if issue_id not in seen_issues: - url = f"https://issues.apache.org/jira/browse/{issue_id}" - links.append({ - 'name': issue_id, - 'url': url - }) - seen_issues.add(issue_id) - - # Remove JIRA IDs from the title. - # Handle multiple patterns: - # 1. "SOLR-12345: description" at the beginning - # 2. "description / SOLR-456: more text" (remove the slash and JIRA ID) - # 3. "/ SOLR-123: description" (leading slash with JIRA) - # 4. Multiple JIRA IDs anywhere in the title - - cleaned_title = title - - # First, clean up leading slash patterns like "/ SOLR-123:" or "/ SOLR-123 " - cleaned_title = re.sub(r'^\s*/\s*(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', '', cleaned_title) - - # Remove slash-separated JIRA IDs in the middle: "... / SOLR-123:" or "... / SOLR-123 " - cleaned_title = re.sub(r'\s+/\s*(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', ' ', cleaned_title) - - # Remove any remaining JIRA IDs at the beginning followed by colon/space - cleaned_title = re.sub(r'^(?:SOLR|LUCENE|INFRA)-\d+[\s:]*', '', cleaned_title) - - # Clean up any remaining leading slashes or extra whitespace - cleaned_title = re.sub(r'^\s*/\s*', '', cleaned_title).strip() - - # Collapse multiple spaces - cleaned_title = re.sub(r'\s+', ' ', cleaned_title) - - return cleaned_title, links + """Return (cleaned_title, links) where links list unique JIRA issues found in title.""" + jira = re.compile(r'(?:SOLR|LUCENE|INFRA)-\d+') + + seen = set() + links = [ + {'name': m, 'url': f'https://issues.apache.org/jira/browse/{m}'} + for m in jira.findall(title) + if not (m in seen or seen.add(m)) + ] + + cleaned = title + # Remove variants at start or when slash-separated, then normalize whitespace + cleaned = re.sub(r'^\s*/\s*' + jira.pattern + r'[\s:]*', '', cleaned) + cleaned = re.sub(r'\s+/\s*' + jira.pattern + r'[\s:]*', ' ', cleaned) + cleaned = re.sub(r'^' + jira.pattern + r'[\s:]*', '', cleaned) + cleaned = re.sub(r'^\s*/\s*', '', cleaned).strip() + cleaned = re.sub(r'\s+', ' ', cleaned) + + return cleaned, links if __name__ == '__main__': From 1ac5622333fe2530a6b782b5c10edb5446bf8056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 16:36:15 +0200 Subject: [PATCH 57/64] Mention task aliases --- dev-docs/changelog.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc index b553a6598b98..057b2e6604bf 100644 --- a/dev-docs/changelog.adoc +++ b/dev-docs/changelog.adoc @@ -77,6 +77,8 @@ user.githubid=johngithub user.asfid=johnapache ---- +TIP: Aliases for `writeChangelog` task are `changelog` and `newChangelog`. + == 4. Writing Good Entries * **Audience** is end-users and administrators, not committers. From c50748c1cf434b784c6be60dcf1f9a2acb26d05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 16:50:28 +0200 Subject: [PATCH 58/64] Refactor changes2html.py to make it more readable and smaller --- .../changes-to-html/changes2html.py | 229 +++++++----------- 1 file changed, 91 insertions(+), 138 deletions(-) diff --git a/gradle/documentation/changes-to-html/changes2html.py b/gradle/documentation/changes-to-html/changes2html.py index d576d99dc4c4..ce302be11bfb 100755 --- a/gradle/documentation/changes-to-html/changes2html.py +++ b/gradle/documentation/changes-to-html/changes2html.py @@ -25,58 +25,62 @@ import sys import re -from datetime import datetime from pathlib import Path class ChangelogParser: """Parse CHANGELOG.md generated by logchange""" + RELEASE_PATTERN = re.compile(r'^\[(\d+(?:\.\d+)*)\]\s*-\s*(.+)$') + SECTION_PATTERN = re.compile(r'^###\s+(\w+(?:\s+\w+)*)\s*(?:\(\d+\s+changes?\))?') + ITEM_PATTERN = re.compile(r'^###|^\[|^- ') + def __init__(self): self.title = "Solr Changelog" self.releases = [] + self.preamble = None + + def _save_section(self, current_release, current_section, current_items): + """Save current section to release if valid""" + if current_release and current_section and current_items: + current_release['sections'].append({ + 'name': current_section, + 'items': current_items + }) def parse(self, content): """Parse CHANGELOG.md content""" lines = content.split('\n') - current_release = None current_section = None current_items = [] - self.preamble = None # Store preamble text - i = 0 + while i < len(lines): line = lines[i] + stripped = line.strip() - # Skip comments at the start - if line.strip().startswith(''): + # Skip HTML comments + if stripped.startswith(''): i += 1 continue # Extract preamble (text before first release) - if not current_release and not self.preamble and line.strip() and not line.startswith('['): - self.preamble = line.strip() + if not current_release and not self.preamble and stripped and not stripped.startswith('['): + self.preamble = stripped i += 1 continue # Match release header: [9.9.0] - 2025-07-24 - match = re.match(r'^\[(\d+(?:\.\d+)*)\]\s*-\s*(.+)$', line) + match = self.RELEASE_PATTERN.match(line) if match: - # Save previous release if exists + self._save_section(current_release, current_section, current_items) if current_release: - if current_section and current_items: - current_release['sections'].append({ - 'name': current_section, - 'items': current_items - }) self.releases.append(current_release) - version = match.group(1) - date = match.group(2).strip() current_release = { - 'version': version, - 'date': date, + 'version': match.group(1), + 'date': match.group(2).strip(), 'sections': [] } current_section = None @@ -85,47 +89,31 @@ def parse(self, content): continue # Match section header: ### Added (9 changes) - match = re.match(r'^###\s+(\w+(?:\s+\w+)*)\s*(?:\(\d+\s+changes?\))?', line) - if match and current_release is not None: - # Save previous section - if current_section and current_items: - current_release['sections'].append({ - 'name': current_section, - 'items': current_items - }) - + match = self.SECTION_PATTERN.match(line) + if match and current_release: + self._save_section(current_release, current_section, current_items) current_section = match.group(1) current_items = [] i += 1 continue - # Match list items: - item text - if line.startswith('- ') and current_release is not None: - # Collect multi-line item - item_text = line[2:] # Remove leading '- ' + # Match list item + if line.startswith('- ') and current_release: + item_text = line[2:] i += 1 - # Collect continuation lines - while i < len(lines): - next_line = lines[i] - if next_line.startswith('- ') or re.match(r'^###', next_line) or re.match(r'^\[', next_line): - break - if next_line.strip(): - item_text += ' ' + next_line.strip() + while i < len(lines) and not self.ITEM_PATTERN.match(lines[i]): + if lines[i].strip(): + item_text += ' ' + lines[i].strip() i += 1 - current_items.append(item_text) continue i += 1 - # Save last release and section + # Save last section and release + self._save_section(current_release, current_section, current_items) if current_release: - if current_section and current_items: - current_release['sections'].append({ - 'name': current_section, - 'items': current_items - }) self.releases.append(current_release) @@ -134,49 +122,36 @@ class HTMLGenerator: JIRA_URL_PREFIX = 'https://issues.apache.org/jira/browse/' GITHUB_PR_PREFIX = 'https://github.com/apache/solr/pull/' + GITHUB_ISSUE_PREFIX = 'https://github.com/apache/solr/issues/' def __init__(self, title="Solr Changelog"): self.title = title self.first_relid = None self.second_relid = None + # Issue extraction patterns: (pattern, prefix, format_string) + self.issue_patterns = [ + (r'\[([A-Z]+-\d+)\]\(https://issues\.apache\.org/jira/browse/\1\)', + self.JIRA_URL_PREFIX, '{0}'), + (r'\[PR#(\d+)\]\(https://github\.com/apache/solr/pull/\1\)', + self.GITHUB_PR_PREFIX, 'PR#{0}'), + (r'\[GITHUB#(\d+)\]\(https://github\.com/apache/solr/issues/\1\)', + self.GITHUB_ISSUE_PREFIX, 'GITHUB#{0}') + ] def extract_issue_from_text(self, text): """ Extract the first JIRA/GitHub issue from markdown text. - Returns (issue_link_html, text_without_issue, authors) + Returns (issue_link_html, text_without_issue) """ - # Extract JIRA issue: [SOLR-123](url) - jira_match = re.search(r'\[([A-Z]+-\d+)\]\(https://issues\.apache\.org/jira/browse/\1\)', text) - if jira_match: - issue_id = jira_match.group(1) - issue_html = f'{issue_id}' - text_before = text[:jira_match.start()] - text_after = text[jira_match.end():] - # Combine the text and remove markdown links from issue part - clean_text = (text_before + text_after).strip() - return issue_html, clean_text, None - - # Extract GitHub PR: [PR#123](url) - pr_match = re.search(r'\[PR#(\d+)\]\(https://github\.com/apache/solr/pull/\1\)', text) - if pr_match: - pr_num = pr_match.group(1) - issue_html = f'PR#{pr_num}' - text_before = text[:pr_match.start()] - text_after = text[pr_match.end():] - clean_text = (text_before + text_after).strip() - return issue_html, clean_text, None - - # Extract GITHUB: [GITHUB#123](url) - gh_match = re.search(r'\[GITHUB#(\d+)\]\(https://github\.com/apache/solr/issues/\1\)', text) - if gh_match: - gh_num = gh_match.group(1) - issue_html = f'GITHUB#{gh_num}' - text_before = text[:gh_match.start()] - text_after = text[gh_match.end():] - clean_text = (text_before + text_after).strip() - return issue_html, clean_text, None - - return None, text, None + for pattern, url_prefix, label_fmt in self.issue_patterns: + match = re.search(pattern, text) + if match: + issue_id = match.group(1) + label = label_fmt.format(issue_id) + issue_html = f'{label}' + text_without = (text[:match.start()] + text[match.end():]).strip() + return issue_html, text_without + return None, text def extract_authors(self, text): """Extract authors from trailing parentheses""" @@ -205,28 +180,20 @@ def format_changelog_item(self, item_text): Format: [ISSUE](url) description (author1) (author2) Output: ISSUE: description
    (authors) """ - # First extract the issue - issue_html, text_after_issue, _ = self.extract_issue_from_text(item_text) + # Extract the issue + issue_html, text_after_issue = self.extract_issue_from_text(item_text) if not issue_html: - # No issue found, just linkify and return return self.linkify_remaining_text(item_text) - # Extract authors + # Extract authors and clean description authors_list, description = self.extract_authors(text_after_issue) - - # Clean up the description - description = description.strip() - # Remove leading punctuation if any description = re.sub(r'^[:\s]+', '', description).strip() - # Build the HTML + # Build HTML html = f'{issue_html}: {self.escape_html(description)}' - if authors_list: - authors_str = ', '.join(authors_list) - html += f'
    ({self.escape_html(authors_str)})' - + html += f'
    ({self.escape_html(", ".join(authors_list))})' return html def linkify_remaining_text(self, text): @@ -502,65 +469,51 @@ def generate_header(self, preamble=None): return html + def _format_section(self, relid, section_name, items): + """Format a single section with items""" + sectid = section_name.lower().replace(' ', '_') + html = [f'
  • ' + f'{self.escape_html(section_name)}'] + html.append(f'   ({len(items)})\n') + html.append(f'
      \n') + for item in items: + html.append(f'
    • {self.format_changelog_item(item)}
      • \n') + html.append('
    \n') + return ''.join(html) + def generate_releases(self, releases): """Generate HTML for releases""" html = [] relcnt = 0 - header = 'h2' - subheader = 'h3' for release in releases: - if relcnt == 2: - header = 'h3' - subheader = 'h4' + version = release.get('version') + if not version: + continue + + relcnt += 1 + if relcnt == 3: html.append('

    Older Releases

    \n') html.append('
    \n') - version = release.get('version') + header = 'h3' if relcnt > 2 else 'h2' + relid = f'v{version}'.replace(' ', '_').lower() date = release.get('date', '') - sections = release.get('sections', []) - - if version: - relcnt += 1 - relid = f'v{version}'.replace(' ', '_').lower() - - html.append(f'<{header}>') - html.append(f'') - html.append('Release ' if version else '') - html.append(f'{self.escape_html(version)} ') - if date: - html.append(f'[{self.escape_html(date)}]') - html.append('') - html.append(f'\n') - html.append(f'
      \n') - - # Render sections - licnt = 0 - for section in sections: - section_name = section.get('name', '') - items = section.get('items', []) - - if section_name: - sectid = section_name.lower().replace(' ', '_') - num_items = len(items) - num_str = f"({num_items})" - - html.append(f'
    • ') - html.append(f'{self.escape_html(section_name)}') - html.append(f'   {num_str}\n') - licnt += 1 - - html.append(f' \n') + html.append('
    \n') if relcnt > 2: html.append('
    \n') From 074d63955389c85df956cb96d34dc2a5d1a2ea97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 16:54:50 +0200 Subject: [PATCH 59/64] Packaging build - include CHANGELOG.md from correct folder --- solr/packaging/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index a6f0cdb259c8..8d0e8723b617 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -80,13 +80,13 @@ distributions { from(rootDir, { include "LICENSE.txt" include "NOTICE.txt" + include "CHANGELOG.md" }) from(project(":solr").projectDir, { include "bin/**" include "licenses/**" exclude "licenses/README.committers.txt" - include "CHANGELOG.md" }) from(projectDir, { From 0ae2fbdcae4e7c665adfb0234388c94cdcfe4df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 17:23:24 +0200 Subject: [PATCH 60/64] Nicer filename for changes yaml files --- dev-tools/scripts/addDepsToChanges.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dev-tools/scripts/addDepsToChanges.py b/dev-tools/scripts/addDepsToChanges.py index 96a5c2ec671d..3286d03cac40 100755 --- a/dev-tools/scripts/addDepsToChanges.py +++ b/dev-tools/scripts/addDepsToChanges.py @@ -94,10 +94,10 @@ def yaml_filename(self) -> str: """ # Clean message for slug slug = self.message.lower() - # Replace spaces with dashes - slug = re.sub(r'\s+', '-', slug) + # Replace whitespace with single space + slug = re.sub(r'\s+', ' ', slug) # Remove non-alphanumeric except dashes - slug = re.sub(r'[^a-z0-9-._]', '', slug) + slug = re.sub(r'[^a-z0-9-._ ]', '', slug) # Calculate available space for slug # Format: "PR" + pr_num + "-" + slug + ".yaml" @@ -109,17 +109,17 @@ def yaml_filename(self) -> str: # Truncate to max length on word boundaries if necessary if len(slug) > max_slug_length: - # Find the last dash within the limit + # Find the last space within the limit truncated = slug[:max_slug_length] - last_dash = truncated.rfind('-') + last_dash = truncated.rfind(' ') if last_dash > max_slug_length // 2: # Keep at least half the available space slug = truncated[:last_dash] else: - # If no good word boundary, use hard limit and clean up trailing dashes - slug = truncated.rstrip('-') + # If no good word boundary, use hard limit and clean up trailing spaces + slug = truncated.rstrip(' ') else: - # Remove trailing dashes - slug = slug.rstrip('-') + # Remove trailing spaces + slug = slug.rstrip(' ') return f"PR{self.pr_num}-{slug}.yaml" From d1a8c4a7c412595efe68e3f7439c337fc29f0062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 18:19:30 +0200 Subject: [PATCH 61/64] changes2logchange.py: Allow spaces in changelog yml files --- dev-tools/scripts/changes2logchange.py | 41 ++++++++++++++++---------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/dev-tools/scripts/changes2logchange.py b/dev-tools/scripts/changes2logchange.py index 20ef3cfe6582..ae6f3f3e1e0d 100755 --- a/dev-tools/scripts/changes2logchange.py +++ b/dev-tools/scripts/changes2logchange.py @@ -297,7 +297,6 @@ class SlugGenerator: # Characters that are unsafe in filenames on various filesystems # Avoid: < > : " / \ | ? * and control characters # Note: # is safe on most filesystems - # Also avoid multiple consecutive dashes UNSAFE_CHARS_PATTERN = re.compile(r'[<>:"/\\|?*\x00-\x1f]+') @staticmethod @@ -308,25 +307,31 @@ def generate_slug(issue_id: str, title: str) -> str: Format: ISSUE-12345-short-slug or VERSION-entry-001-short-slug Uses the actual issue ID without forcing SOLR- prefix Ensures filesystem-safe filenames and respects word boundaries + Whitespace is preserved as spaces (not converted to dashes) """ # Sanitize issue_id to remove unsafe characters (preserve case and # for readability) base_issue = SlugGenerator._sanitize_issue_id(issue_id) - # Create slug from title: lowercase, replace unsafe chars with dash + # Create slug from title: lowercase, preserve spaces, replace only unsafe chars with dash title_slug = SlugGenerator._sanitize_filename_part(title) # Limit to reasonable length while respecting word boundaries # Target max length: 50 chars for slug (leaving room for base_issue and dash) if len(title_slug) > 50: - # Find last word boundary within 50 chars + # Find last word/space boundary within 50 chars truncated = title_slug[:50] - # Find the last dash (word boundary) - last_dash = truncated.rfind('-') - if last_dash > 20: # Keep at least 20 chars to avoid too-short slugs - title_slug = truncated[:last_dash] + # Find the last space within the limit + last_space = truncated.rfind(' ') + if last_space > 20: # Keep at least 20 chars to avoid too-short slugs + title_slug = truncated[:last_space] else: - # If no good word boundary found, use hard limit - title_slug = truncated.rstrip('-') + # If no good space boundary, try to find a dash (from unsafe chars) + last_dash = truncated.rfind('-') + if last_dash > 20: + title_slug = truncated[:last_dash] + else: + # If no good boundary, use hard limit and clean up + title_slug = truncated.rstrip(' -') return f"{base_issue}-{title_slug}" @@ -356,23 +361,27 @@ def _sanitize_filename_part(text: str) -> str: Sanitize text for use in filenames. - Convert to lowercase - Replace unsafe characters with dashes - - Remove multiple consecutive dashes - - Strip leading/trailing dashes + - Convert any whitespace to space (preserved in filename) + - Remove multiple consecutive spaces or dashes + - Strip leading/trailing spaces and dashes """ # Convert to lowercase text = text.lower() + # Normalize all whitespace to single spaces + text = re.sub(r'\s+', ' ', text) + # Replace unsafe characters with dash text = SlugGenerator.UNSAFE_CHARS_PATTERN.sub('-', text) - # Replace non-alphanumeric (except dash) with dash - text = re.sub(r'[^a-z0-9-]+', '-', text) + # Replace other non-alphanumeric (except space and dash) with dash + text = re.sub(r'[^a-z0-9\s-]+', '-', text) - # Replace multiple consecutive dashes with single dash + # Replace multiple consecutive dashes with single dash (but preserve spaces) text = re.sub(r'-+', '-', text) - # Strip leading/trailing dashes - text = text.strip('-') + # Strip leading/trailing spaces and dashes + text = text.strip(' -') return text From 2c68693ece6e41360177f66a6c9c64501f706ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 18:43:10 +0200 Subject: [PATCH 62/64] Add a changelog yml to the PR Add validation of changelog files Fix some comments by Copilot --- .github/scripts/validate-changelog-yaml.py | 113 ++++++++++++++++++ .github/workflows/validate-changelog.yml | 41 ++++++- ...Use logchange for changelog management.yml | 7 ++ 3 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/validate-changelog-yaml.py create mode 100644 changelog/unreleased/SOLR-17619 Use logchange for changelog management.yml diff --git a/.github/scripts/validate-changelog-yaml.py b/.github/scripts/validate-changelog-yaml.py new file mode 100644 index 000000000000..be1c9205cdb5 --- /dev/null +++ b/.github/scripts/validate-changelog-yaml.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Validates changelog YAML files in changelog/unreleased/ folder. + +Checks: +- File is valid YAML +- Contains required 'title' field (non-empty string) +- Contains required 'type' field (one of: added, changed, fixed, deprecated, removed, dependency_update, security, other) +- Contains required 'authors' field with at least one author +- Each author has a 'name' field (non-empty string) +""" + +import sys +import yaml + + +def validate_changelog_yaml(file_path): + """Validate a changelog YAML file.""" + valid_types = ['added', 'changed', 'fixed', 'deprecated', 'removed', 'dependency_update', 'security', 'other'] + + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = yaml.safe_load(f) + + # Check if file contains a mapping (dictionary) + if not isinstance(data, dict): + print(f"::error file={file_path}::File must contain YAML mapping (key-value pairs)") + return False + + # Validate 'title' field + if 'title' not in data or not data['title']: + print(f"::error file={file_path}::Missing or empty 'title' field") + return False + + if not isinstance(data['title'], str) or not data['title'].strip(): + print(f"::error file={file_path}::Field 'title' must be a non-empty string") + return False + + # Validate 'type' field + if 'type' not in data or not data['type']: + print(f"::error file={file_path}::Missing or empty 'type' field") + return False + + if data['type'] not in valid_types: + print(f"::error file={file_path}::Invalid 'type': '{data['type']}'. Must be one of: {', '.join(valid_types)}") + return False + + # Validate 'authors' field + if 'authors' not in data or not data['authors']: + print(f"::error file={file_path}::Missing or empty 'authors' field") + return False + + if not isinstance(data['authors'], list): + print(f"::error file={file_path}::Field 'authors' must be a list") + return False + + if len(data['authors']) == 0: + print(f"::error file={file_path}::Field 'authors' must contain at least one author") + return False + + # Validate each author + for i, author in enumerate(data['authors']): + if not isinstance(author, dict): + print(f"::error file={file_path}::Author {i} must be a mapping (key-value pairs)") + return False + if 'name' not in author or not author['name']: + print(f"::error file={file_path}::Author {i} missing or empty 'name' field") + return False + if not isinstance(author['name'], str) or not author['name'].strip(): + print(f"::error file={file_path}::Author {i} 'name' must be a non-empty string") + return False + + # All validations passed + print(f"✓ {file_path} is valid") + print(f" Title: {data['title']}") + print(f" Type: {data['type']}") + print(f" Authors: {', '.join(a['name'] for a in data['authors'])}") + return True + + except yaml.YAMLError as e: + print(f"::error file={file_path}::Invalid YAML: {e}") + return False + except Exception as e: + print(f"::error file={file_path}::Error validating file: {e}") + return False + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: validate-changelog-yaml.py ") + sys.exit(1) + + file_path = sys.argv[1] + if not validate_changelog_yaml(file_path): + sys.exit(1) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 24582428c9fe..6969c9cb9824 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -54,7 +54,7 @@ jobs: CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) # Check if any files were added to changelog/unreleased/ - if echo "$CHANGED_FILES" | grep -q "^solr/changelog/unreleased/"; then + if echo "$CHANGED_FILES" | grep -q "^changelog/unreleased/"; then echo "✓ Changelog entry found" exit 0 fi @@ -63,7 +63,7 @@ jobs: HAS_NON_DOCS_CHANGES=false while IFS= read -r file; do # Skip changelog, docs, tests, and certain config files - if ! echo "$file" | grep -qE "(^solr/changelog/|^solr/solr-ref-guide/|^dev-docs/|\.md$|\.adoc$|^solr/.*/test|\.gradle$|\.properties$|README|NOTICE|LICENSE)"; then + if ! echo "$file" | grep -qE "(^changelog/|^solr/solr-ref-guide/|^dev-docs/|\.md$|\.adoc$|^solr/.*/test|\.gradle$|\.properties$|README|NOTICE|LICENSE)"; then HAS_NON_DOCS_CHANGES=true break fi @@ -86,3 +86,40 @@ jobs: echo "If this PR should not have a changelog entry (e.g., refactoring, internal cleanup)," echo "add the 'no-changelog' label to this PR." exit 1 + + - name: Validate changelog YAML structure + if: steps.check-label.outputs.skip == 'false' + run: | + # Get the list of changed files + CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) + + # Find all YAML files added to changelog/unreleased/ + YAML_FILES=$(echo "$CHANGED_FILES" | grep "^solr/changelog/unreleased/.*\.ya\?ml$" || true) + + if [ -z "$YAML_FILES" ]; then + exit 0 + fi + + echo "Validating changelog YAML files..." + VALIDATION_FAILED=false + + while IFS= read -r file; do + if [ -z "$file" ]; then + continue + fi + + echo "" + echo "Validating: $file" + + # Validate using a Python script + python3 .github/scripts/validate-changelog-yaml.py "$file" + + if [ $? -ne 0 ]; then + VALIDATION_FAILED=true + fi + + done <<< "$YAML_FILES" + + if [ "$VALIDATION_FAILED" = true ]; then + exit 1 + fi diff --git a/changelog/unreleased/SOLR-17619 Use logchange for changelog management.yml b/changelog/unreleased/SOLR-17619 Use logchange for changelog management.yml new file mode 100644 index 000000000000..63a14ab5278c --- /dev/null +++ b/changelog/unreleased/SOLR-17619 Use logchange for changelog management.yml @@ -0,0 +1,7 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Use logchange for changelog management +type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Jan Høydahl + nick: janhoy + url: https://home.apache.org/phonebook.html?uid=janhoy From 265c1560d01af30715899e0689c35da3c1ad55b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 19:04:58 +0200 Subject: [PATCH 63/64] Fixes in changelog validation workflow Review changes --- .github/workflows/validate-changelog.yml | 7 ++++++- changelog/README.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 6969c9cb9824..87efd9cfe05b 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -94,7 +94,7 @@ jobs: CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) # Find all YAML files added to changelog/unreleased/ - YAML_FILES=$(echo "$CHANGED_FILES" | grep "^solr/changelog/unreleased/.*\.ya\?ml$" || true) + YAML_FILES=$(echo "$CHANGED_FILES" | grep "^changelog/unreleased/.*\.ya\?ml$" || true) if [ -z "$YAML_FILES" ]; then exit 0 @@ -121,5 +121,10 @@ jobs: done <<< "$YAML_FILES" if [ "$VALIDATION_FAILED" = true ]; then + echo "::error::This PR contains an invalid changelog entry." + echo "" + echo "Please ensure it is valid YAML and complies to" + echo "the specification in dev-docs/changelog.adoc." + exit 1 fi diff --git a/changelog/README.md b/changelog/README.md index 9947e6006284..3686afdfec07 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -18,4 +18,4 @@ We are in the process of migrating to a new way of managing our changelog. Please see [dev-docs/changelog.adoc](../dev-docs/changelog.adoc) for details. -In a transition period it is still possible to merge your changelog entry to `solr/CHANGES.txt`, but on Wednesday October 29th this will not longer be the case. +In a transition period it is still possible to merge your changelog entry to `solr/CHANGES.txt`, but then you can only use the new process. From 9d160cb41ef60022b6ea2508fd1fbbf75e034002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Oct 2025 19:24:54 +0200 Subject: [PATCH 64/64] No need to print more than a link to changelog doc --- .github/workflows/validate-changelog.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/validate-changelog.yml b/.github/workflows/validate-changelog.yml index 87efd9cfe05b..e5b7cf158ac3 100644 --- a/.github/workflows/validate-changelog.yml +++ b/.github/workflows/validate-changelog.yml @@ -121,10 +121,7 @@ jobs: done <<< "$YAML_FILES" if [ "$VALIDATION_FAILED" = true ]; then - echo "::error::This PR contains an invalid changelog entry." - echo "" - echo "Please ensure it is valid YAML and complies to" - echo "the specification in dev-docs/changelog.adoc." + echo "Please see dev-docs/changelog.adoc for more info." exit 1 fi