From ceb45ef5f4df6d466a4df1920734e9e1f1fa2ad4 Mon Sep 17 00:00:00 2001 From: Mendon Kissling <59585235+mendonk@users.noreply.github.com> Date: Fri, 13 Mar 2026 13:09:41 -0400 Subject: [PATCH 1/5] new-product-name --- antora.yml | 4 ++-- modules/ROOT/pages/faqs.adoc | 5 +++++ modules/ROOT/pages/index.adoc | 10 +++++++--- modules/ROOT/partials/luna-name.adoc | 4 ++++ 4 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 modules/ROOT/partials/luna-name.adoc diff --git a/antora.yml b/antora.yml index a499de2..b1cf735 100644 --- a/antora.yml +++ b/antora.yml @@ -1,6 +1,6 @@ name: 'luna-streaming' version: '2.10-2.x' -title: 'Luna Streaming' +title: 'IBM Elite Support for Apache Pulsar' start_page: 'index.adoc' nav: @@ -8,7 +8,7 @@ nav: asciidoc: attributes: - product: 'Luna Streaming' + product: 'IBM Elite Support for Apache Pulsar' luna-version: '2.10' pulsar-version: '2.10' admin-console-version: '2.0.4' diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index fa49441..18ec9a2 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -7,6 +7,11 @@ If you are new to {company} {product} and its {pulsar} enhancements, these FAQs {company} {product} is a new Kubernetes-based distribution of {pulsar}, based on the technology that https://kesque.com/[Kesque] built to run its {pulsar-short}-as-a-service. +== Are {product} and {company} Luna Streaming the same offering? + +Yes. +include::partial$luna-name.adoc[] + == What components and features are provided by {company} {product}? In addition to {pulsar} itself, {company} {product} provides: diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index bfaa5ce..342d9e4 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -1,11 +1,15 @@ -= Welcome to {company} {product} += Welcome to {product} (formerly {company} Luna Streaming) :navtitle: {product} -{company} {product} is a production-ready distribution of {pulsar} built to run seamlessly on any CNCF conformant version of Kubernetes. {company} {product} provides all of the core capabilities included in the Apache Community version of {pulsar}, plus a number of additional tools and features to facilitate administration and operational tasks associated with running {pulsar} in production. +{product} is a production-ready distribution of {pulsar} built to run seamlessly on any CNCF conformant version of Kubernetes. + +{product} provides all of the core capabilities included in the Apache Community version of {pulsar}, plus a number of additional tools and features to facilitate administration and operational tasks associated with running {pulsar} in production. + +include::partial$luna-name.adoc[] == Release notes -The latest release of {company} {product} is {luna-version}, which matches the supported, distributed {pulsar-short} version numbers. +The latest release of {product} is {luna-version}, which matches the supported, distributed {pulsar-short} version numbers. The prior {product} release (numbered 1.0.x or 2.7.2) provided the {pulsar-short} 2.7.2 distribution. diff --git a/modules/ROOT/partials/luna-name.adoc b/modules/ROOT/partials/luna-name.adoc new file mode 100644 index 0000000..0c961e8 --- /dev/null +++ b/modules/ROOT/partials/luna-name.adoc @@ -0,0 +1,4 @@ +{product} is the new name for Luna Streaming after IBM acquired DataStax. +In this documentation, {product} and Luna Streaming refer to the same product. + +For technical specifications, inclusions, and exclusions of the {product} offering, see the https://www.ibm.com/docs/en/supportforpulsar?topic=elite-support-pulsar[{product} page in the IBM documentation]. \ No newline at end of file From 55d36542d5e5366bfff7746c6528e241336a52d8 Mon Sep 17 00:00:00 2001 From: Mendon Kissling <59585235+mendonk@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:35:05 -0400 Subject: [PATCH 2/5] peer-review --- README.adoc | 6 +- modules/ROOT/pages/faqs.adoc | 49 +- modules/ROOT/pages/index.adoc | 4 +- .../pages/admin-console-tutorial.adoc | 4 +- modules/components/pages/pulsar-beam.adoc | 7 +- modules/components/pages/pulsar-sql.adoc | 7 +- modules/connectors/pages/index.adoc | 13 +- .../pages/production-cluster-sizing.adoc | 238 +++++++++ .../pages/quickstart-helm-installs.adoc | 9 +- .../pages/quickstart-server-installs.adoc | 11 +- .../pages/supported-versions.adoc | 94 ++++ .../install-upgrade/pages/upgrade-guide.adoc | 454 ++++++++++++++++++ .../operations/pages/functions-transform.adoc | 2 +- modules/operations/pages/functions.adoc | 8 +- modules/operations/pages/scale-cluster.adoc | 7 +- 15 files changed, 868 insertions(+), 45 deletions(-) create mode 100644 modules/install-upgrade/pages/production-cluster-sizing.adoc create mode 100644 modules/install-upgrade/pages/supported-versions.adoc create mode 100644 modules/install-upgrade/pages/upgrade-guide.adoc diff --git a/README.adoc b/README.adoc index 1dd0d3c..371f651 100644 --- a/README.adoc +++ b/README.adoc @@ -1,7 +1,7 @@ -= {company} {product} Docs += {product} Docs (formerly {company} Luna Streaming) // Variables: :company: DataStax -:product: Luna Streaming +:product: IBM Elite Support for Apache Pulsar :repo-name: pulsar-docs :github-org: datastax // Settings: @@ -32,7 +32,7 @@ endif::[] // External URLs: :asciidoc-language: https://docs.asciidoctor.org/asciidoc/latest/ -This repository contains the source files for the {company} {product} documentation. +This repository contains the source files for the {product} documentation. toc::[] diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index 18ec9a2..e2406b4 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -1,20 +1,20 @@ = {product} FAQs :navtitle: FAQs -If you are new to {company} {product} and its {pulsar} enhancements, these FAQs are for you. +If you are new to {product} and its {pulsar} enhancements, these FAQs are for you. -== What is {company} {product}? +== What is {product}? -{company} {product} is a new Kubernetes-based distribution of {pulsar}, based on the technology that https://kesque.com/[Kesque] built to run its {pulsar-short}-as-a-service. +{product} is a Kubernetes-based distribution of {pulsar}, based on the technology that https://kesque.com/[Kesque] built to run its {pulsar-short}-as-a-service. == Are {product} and {company} Luna Streaming the same offering? Yes. include::partial$luna-name.adoc[] -== What components and features are provided by {company} {product}? +== What components and features are provided by {product}? -In addition to {pulsar} itself, {company} {product} provides: +In addition to {pulsar} itself, {product} provides: * An installer that can stand up a dev or production cluster on bare metal or VMs without a pre-existing Kubernetes environment * A Helm chart that can deploy and manage {pulsar-short} on your current Kubernetes infrastructure @@ -22,24 +22,24 @@ In addition to {pulsar} itself, {company} {product} provides: * A management dashboard * A monitoring and alerting system -== On which version of {pulsar} is {company} {product} based? +== On which version of {pulsar} is {product} based? -{company} {product} {luna-version} is based on its distribution of {pulsar} {pulsar-version}, plus features and additional enhancements from {company} contributors. +{product} {luna-version} is based on its distribution of {pulsar} {pulsar-version}, plus features and additional enhancements from {company} contributors. -== What does {company} {product} provide that I cannot get with open-source {pulsar}? +== What does {product} provide that I cannot get with open-source {pulsar}? -{company} {product} is a hardened version of {pulsar} that been run through additional testing to ensure it is ready for production use. It also includes additional tooling to help monitor your system, including an enhanced Admin Console and a Heartbeat service to monitor the system health. +{product} is a hardened version of {pulsar} that been run through additional testing to ensure it is ready for production use. It also includes additional tooling to help monitor your system, including an enhanced Admin Console and a Heartbeat service to monitor the system health. -== Is {company} {product} an open-source project? +== Is {product} an open-source project? -Yes, {company} {product} is open source. See the <>. +Yes, {product} is open source. See the <>. -== Which Kubernetes platforms are supported by {company} {product}? +== Which Kubernetes platforms are supported by {product}? They include Minikube, K8d, Kind, Google Kubernetes Engine (GKE), Microsoft Azure Kubernetes Service (AKS), Amazon Elastic Kubernetes Service (EKS), and other commonly used platforms. [#gitHubRepos] -== Where are the {company} {product} public GitHub repos? +== Where are the {product} public GitHub repos? There are several public repos, each with a different purpose. See: @@ -50,25 +50,26 @@ There are several public repos, each with a different purpose. See: * {pulsar-sink-repo}[{pulsar-sink-repo}] : This is the {company} {pulsar} Connector (`pulsar-sink`) repo. * https://github.com/datastax/burnell[https://github.com/datastax/burnell] : This is a utility for {pulsar-short} that provides various functions, such as key initialization for authentication, and JWT token creation API. -== Is there a prerequisite version of Java needed for the {company} {product} installation? +== Is there a prerequisite version of Java needed for the {product} installation? -The {company} {product} distribution is designed for Java 11. However, because the product releases Docker images, you do not need to install Java (8 or 11) in advance. Java 11 is bundled in the Docker image. +The {product} distribution is designed for Java 17. However, because the product releases Docker images, you do not need to install Java 17 in advance. Java 17 is bundled in the Docker image. -== What are the install options for {company} {product}? +== What are the install options for {product}? -* Use the Helm chart provided at {pulsar-helm-chart-repo}[{pulsar-helm-chart-repo}] to install {company} {product} in an existing Kubernetes cluster on your laptop or hosted by a cloud provider. -* Use the tarball provided at {pulsar-repo}/releases[{pulsar-repo}/releases] to install {company} {product} on a server or VM. -* Use the {company} Ansible scripts provided at {pulsar-ansible-repo}[{pulsar-ansible-repo}] to install {company} {product} on a server or VM with our provided playbooks. +* **{kaap-short} Helm chart (Recommended)**: Use the {company} {kaap-operator-repo}[{kaap-short} Helm chart], which provides Kubernetes-native autoscaling and simplified management for {pulsar} clusters. For more information, see the xref:kaap-operator::index.adoc[{kaap}] documentation. +* **{product} Helm chart**: Use the Helm chart provided at {pulsar-helm-chart-repo}[{pulsar-helm-chart-repo}] to install {product} in an existing Kubernetes cluster on your laptop or hosted by a cloud provider. +* **Tarball**: Use the tarball provided at {pulsar-repo}/releases[{pulsar-repo}/releases] to install {product} on a server or VM. +* **Ansible**: Use the {company} Ansible scripts provided at {pulsar-ansible-repo}[{pulsar-ansible-repo}] to install {product} on a server or VM with our provided playbooks. -== How do I install {company} {product} in my Kubernetes cluster? +== How do I install {product} in my Kubernetes cluster? Follow the full instructions in xref:install-upgrade:quickstart-helm-installs.adoc[Quick Start for Helm Chart installs]. -== How do I install {company} {product} on my server or VM? +== How do I install {product} on my server or VM? Follow the full instructions in xref:install-upgrade:quickstart-server-installs.adoc[Quick Start for Server/VM installs]. -== What task can I perform in the {company} {product} Admin Console? +== What task can I perform in the {product} Admin Console? From the Admin Console, you can: @@ -100,7 +101,7 @@ The {company} {pulsar} Connector allows single-record acknowledgement and negati There are two packages: -* The `pulsar-sink` functionality of {company} {pulsar} Connector is included with {company} {product}. It's built in! +* The `pulsar-sink` functionality of {company} {pulsar} Connector is included with {product}. It's built in! * You can optionally download the {company} {pulsar} Connector `.nar` file from the {pulsar-sink-repo}/releases[`pulsar-sink` repository], and then use it as its own product with your open-source {pulsar} install. If you're using open-source software (OSS) {pulsar}, you can use {company} {pulsar} Connector with the OSS to take advantage of this `pulsar-sink` for {cass-short}. See the xref:pulsar-connector:ROOT:index.adoc[{company} {pulsar} Connector documentation]. @@ -110,6 +111,6 @@ If you're using open-source software (OSS) {pulsar}, you can use {company} {puls This source connector streams data changes from {cass-short} tables to {pulsar-short} topics. For more information, see the xref:cdc-for-cassandra:ROOT:index.adoc[{company} CDC for {cass-short} connector documentation]. -== What client APIs does {company} {product} provide? +== What client APIs does {product} provide? The same as for {pulsar}. See https://pulsar.apache.org/docs/en/client-libraries/. \ No newline at end of file diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index 342d9e4..3d0c0ef 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -13,11 +13,11 @@ The latest release of {product} is {luna-version}, which matches the supported, The prior {product} release (numbered 1.0.x or 2.7.2) provided the {pulsar-short} 2.7.2 distribution. -Refer to the {company} {product} https://github.com/datastax/release-notes/blob/master/Luna_Streaming_2.8_Release_Notes.md[release notes], which are hosted in our public GitHub repo, for information & linked commit IDs that were implemented in the latest {product} {luna-version} release. +Refer to the {product} https://github.com/datastax/release-notes/blob/master/Luna_Streaming_3.1_Release_Notes.md[release notes], which are hosted in our public GitHub repo, for information & linked commit IDs that were implemented in the latest {product} {luna-version} release. == Components -In addition to the distribution of https://pulsar.apache.org/en/versions/[{pulsar} {pulsar-version}], {company} {product} provides: +In addition to the distribution of https://pulsar.apache.org/en/versions/[{pulsar} {pulsar-version}], {product} provides: * A xref:install-upgrade:quickstart-helm-installs.adoc[Helm chart] that deploys and manages {pulsar-short} on your current CNCF-conformant Kubernetes infrastructure diff --git a/modules/components/pages/admin-console-tutorial.adoc b/modules/components/pages/admin-console-tutorial.adoc index 5da1992..c50b58a 100644 --- a/modules/components/pages/admin-console-tutorial.adoc +++ b/modules/components/pages/admin-console-tutorial.adoc @@ -1,11 +1,11 @@ = {pulsar-short} Admin Console -The *{company} Admin Console for {pulsar-reg}* is a web-based UI from {company} that administers topics, namespaces, sources, sinks, and various aspects of {pulsar} features. +The *Admin Console for {pulsar-reg}* is a web-based UI from {company} that administers topics, namespaces, sources, sinks, and various aspects of {pulsar} features. [#getting-started] == Getting Started in {pulsar-short} Admin Console -In the *{product} {pulsar-short} Admin Console*, you can use {pulsar-short} clients to send and receive pub/sub messages. +In the *{pulsar-short} Admin Console*, you can use {pulsar-short} clients to send and receive pub/sub messages. If you installed the Admin console with the xref:install-upgrade:quickstart-helm-installs.adoc[{company} Helm chart], access the Admin console with the `pulsar-adminconsole` external load balancer endpoint in your cloud provider: diff --git a/modules/components/pages/pulsar-beam.adoc b/modules/components/pages/pulsar-beam.adoc index 1344508..d9e7e0e 100644 --- a/modules/components/pages/pulsar-beam.adoc +++ b/modules/components/pages/pulsar-beam.adoc @@ -7,7 +7,12 @@ The {pulsar-beam-repo}[{pulsar-beam}] project is an HTTP-based streaming and que With {pulsar-beam}, you can send messages over HTTP, push messages to a webhook or cloud function, chain webhooks and functions together, or stream messages through server-sent events (SSE). -In this guide, you'll install a minimal {company} {pulsar-short} Helm chart that includes {pulsar-beam}. +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +In this guide, you'll install a minimal {product} Helm chart that includes {pulsar-beam}. == Prerequisites diff --git a/modules/components/pages/pulsar-sql.adoc b/modules/components/pages/pulsar-sql.adoc index d08026e..4d572ab 100644 --- a/modules/components/pages/pulsar-sql.adoc +++ b/modules/components/pages/pulsar-sql.adoc @@ -9,7 +9,12 @@ Stream processing, real-time analytics, and highly customized dashboards are jus {pulsar-short} offers a pre-made plugin for Trino that is included in its distribution. Additionally, {pulsar-short} has built-in options to create Trino workers and automatically configure the communications between {pulsar-short}'s ledger and Trino. -In this guide, we will use the {company} {pulsar-short} Helm Chart to install a {pulsar-short} cluster with {pulsar-short} SQL. +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +In this guide, we will use the {product} Helm Chart to install a {pulsar-short} cluster with {pulsar-short} SQL. The Trino coordinator and desired number of workers will be created directly in the cluster. == Prerequisites diff --git a/modules/connectors/pages/index.adoc b/modules/connectors/pages/index.adoc index cfe4e8e..e01501a 100644 --- a/modules/connectors/pages/index.adoc +++ b/modules/connectors/pages/index.adoc @@ -2,11 +2,16 @@ :navtitle: {pulsar-short} I/O connectors reference :page-aliases: operations:io-connectors.adoc +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + {product} offers fully-managed versions of https://pulsar.apache.org/docs/en/io-overview/[{pulsar-reg} sink and source connectors]. [TIP] ==== -There are three versions of the {company} {product} distribution. +There are three versions of the {product} distribution. The `lunastreaming-all` version includes all connectors. ==== @@ -45,13 +50,13 @@ The following source connectors are included in {product}: You can use the {web-ui} and the `xref:components:admin-console-tutorial.adoc[pulsar-admin]` CLI to create, monitor, and manage sink and source connectors. -Although you use the same base commands to create and update all {product} {pulsar-short} connectors, each connector has different configuration options. +Although you use the same base commands to create and update all {pulsar-short} connectors, each connector has different configuration options. For example commands and configuration details, see the documentation for your preferred <> and <>. === `pulsar-admin` CLI sink operations Get available sink connectors:: -Get a list of sink connectors that are available in your {product} {pulsar-short} tenant: +Get a list of sink connectors that are available in your {pulsar-short} tenant: + [source,shell] ---- @@ -136,7 +141,7 @@ Delete a sink connector:: === `pulsar-admin` CLI source operations Get available source connectors:: -Get a list of source connectors that are available in your {product} {pulsar-short} tenant: +Get a list of source connectors that are available in your {pulsar-short} tenant: + [source,shell] ---- diff --git a/modules/install-upgrade/pages/production-cluster-sizing.adoc b/modules/install-upgrade/pages/production-cluster-sizing.adoc new file mode 100644 index 0000000..1ffad6a --- /dev/null +++ b/modules/install-upgrade/pages/production-cluster-sizing.adoc @@ -0,0 +1,238 @@ += Production Cluster Sizing + +This document summarizes {company}'s recommendations for the sizing and optimization of an {pulsar} cluster on Linux in a production environment. +Remember, a {pulsar-short} *instance* is made of one or many clusters. + +Of course, the sizing of a cluster depends on factors like use case and expected load, so this document is not intended to be a one-size-fits-all guide. Rather, we'd like to demonstrate how we consider and spec the initial size of a {pulsar-short} cluster, and assist you on your journey to unlocking the scaling power of {pulsar-short}. + +This page summarizes the requirements, assumptions, definitions, and methodologies that inform our cluster sizing recommendations. +If you're looking for specific deployment topology recommendations, see xref:cluster-sizing-reference.adoc[]. + +include::operations:partial$operator-scaling.adoc[] + +== Dedicated VMs or Kubernetes + +As you begin your journey to design an {pulsar} cluster, one of the first questions to consider is what infrastructure your cluster will run on. +Most of this guide will focus on running a cluster with dedicated virtual machines. +While Kubernetes is the more popular option, it is easier to express disk calculations, throughput, and secure communications in terms of a VM. + +== {pulsar-short} cluster components + +{pulsar-short} clusters come in many shapes and sizes. There are minimum components for base functionality, and there are recommended components that make message routing, management, and observability easier. For this guide we will focus on the required components and what it takes to make them resilient to outages and highly available in a three-zone cloud. + +=== Required components + +* https://pulsar.apache.org/docs/concepts-architecture-overview/#metadata-store[{zookeeper-reg}] - This is {pulsar-short}'s meta data store. It stores data about a cluster's configuration, helps the proxy direct messages to the correct broker, and holds bookie configurations. Start with 1 instance of {zookeeper-short} in each availability zone (AZ) to mitigate a single failure point, and scale {zookeeper-short} as cluster traffic increases. You could scale {zookeeper-short} as traffic within the cluster increases, but it shouldn't be very often as it can handle quite a bit of load. + +* https://pulsar.apache.org/docs/concepts-architecture-overview/#brokers[Broker] - This is {pulsar-short}'s message router. +Ideally, each broker should be fully utilized without becoming a performance bottleneck. +The {pulsar-short} broker is stateless, so it requires considerable computing power but not much storage. +Start with 1 broker instance in each zone, and set a scaling rule that watches CPU load. +The best way to optimize this is through performance testing based on your cluster's workload characteristics. + +* https://pulsar.apache.org/docs/concepts-architecture-overview/#apache-bookkeeper[{bookkeeper-reg} (bookie)] - This is {pulsar-short}'s data store. +{bookkeeper-short} stores message data in a low-latency, resilient way. +{pulsar-short} uses {bookkeeper-short}'s quorum math to function, so a loss of 1 {bookkeeper-short} instance won't bring your system down, but will cause some data loss. +Start with at least 3 bookies, with 1 in each AZ. +At least 2 bookies per AZ are required for high availability, so if one bookie goes down, the other bookie in the AZ can take over. +Scale bookies up on disc usage percentage. Scale down manually by making a bookie read-only, offloading its data, then terminating the instance. + +[#recommended] +=== Recommended server components + +The {product} Helm chart deployment includes optional but highly recommended server components for better {pulsar-short} cluster metrics monitoring and operation visibility. + +* https://bookkeeper.apache.org/docs/admin/autorecovery[{bookkeeper-short} AutoRecovery] - This is a {pulsar-short} component that recovers {bookkeeper-short} data in the event of a bookie outage. While optional you will want the insurance of AutoRecovery working on your behalf. +A single instance of AutoRecovery should be adequate - only in the most heavily-used clusters will you need more. +* https://pulsar.apache.org/docs/concepts-architecture-overview/#pulsar-proxy[{pulsar-short} proxy] - The {pulsar-short} proxy is just that - a proxy. +It runs at the edge of the cluster with public facing endpoints. +Without it, your brokers would expose those endpoints, which is not an ideal configuration in production. +{pulsar-short} proxy also offers special options for cluster extensions, like the xref:components:starlight.adoc[{company} Starlight suite of APIs]. +Start with a proxy in each zone. +The proxy will be made aware of all the brokers in the same zone and load balance across them. +Have your load balancer round-robin to all proxy instances in all zones. +Proxy is optional for VM deployments and required for Kubernetes deployments. +Scale proxies by their network load or (if running extensions) also scale on CPU usage. +* https://pulsar.apache.org/docs/functions-worker-run-separately/[Dedicated functions worker(s)] - You can optionally run dedicated function workers in a {pulsar-short} cluster. +Without dedicated function workers, functions run as a separate process on the broker. +Function worker spec is usually focused on compute and memory. +Scale the workers based on overall usage (both CPU and memory). +* xref:luna-streaming:components:admin-console-tutorial.adoc[{pulsar-short} AdminConsole] - This is an optional web-based admin console for managing {pulsar-short} clusters, and makes management much easier than tons of CLI commands. The sizing and scaling for AdminConsole has nothing to do with the cluster, as it is not a failure point. +* xref:luna-streaming:components:heartbeat-vm.adoc[{pulsar-short} Heartbeat] - This is an optional component that monitors the health of {pulsar-short} cluster and emits metrics about the cluster that are helpful for observing and debugging issues. +* Prometheus/Grafana/Alert manager stack - This is the default observability stack for a cluster. The Luna Helm chart includes pre-made dashboards in Grafana and pre-wires all the metrics scraping. + +image::pulsar-components.png[] + +[#message-retention] +== Message retention + +The broker ensures messages are received and delivered appropriately, but it is a stateless process so it doesn't use its memory to track this. Instead, the broker uses {bookkeeper-short}s (bookies) to store message data and the message's acknowledgement state. +A great benefit of {bookkeeper-short} is its quorum policies. These policies make each bookie aware of the other bookies to form a {bookkeeper-short} cluster. With a cluster established, the cluster can have acknowledgement rules that form a data replication factor. For example, if you had 3 bookies in a {bookkeeper-short} cluster with an acknowledgement rule that at least 2 of the 3 bookies must have a copy of the data, then the cluster has a replication factor of 2. A {pulsar-short} broker uses the `managedLedgerDefaultAckQuorum` and `managedLedgerDefaultWriteQuorum` configurations to set the bounds of this rule. For more about {bookkeeper-short} persistence, see https://pulsar.apache.org/docs/administration-zk-bk/#bookkeeper-persistence-policies[here]. + +When a client produces a message, the broker will not acknowledge receipt until the replication factor has been achieved. Continuing from the above example, if the replication factor is 2, a broker's acknowledgment means a minimum of 2 bookies have confirmed storage of message data. If the broker times out waiting for at least 2 responses from the bookies, then the broker will not acknowledge receipt with the client. The client will need to handle the exception by attempting to resend or fail. This process forms one of {pulsar-short}'s core values - guaranteed message receipt. + +Now that the broker has a message, it guarantees delivery to the associated topic's subscribers. We refer to this as the broker's backlog. The size of the backlog is sometimes expressed by the number of messages. For example, a {pulsar-short} operator might say, "we try to keep the backlog below 100 messages." The number of brokers available to process messages directly impacts the size of the backlog. However, the number of messages is not a meaningful number on its own without knowing the size of the messages. Message size is essential information because it determines how full a bookie's disk will be. If the backlog has 100 messages that are 4Gb each, then 400Gb is occupied on a bookie's disk. If the backlog has 100 messages that are 1Kb each, then only 100Kb is occupied on the bookie's disk. Quite a difference in storage capacity! + +Until all subscribers have acknowledged receipt of a message, the broker will not mark a message as acknowledged. This is another core feature of {pulsar-short} - guaranteed message delivery. But there are realities around this - we must assume that all functions, sinks, and clients subscribed to the message's topic are healthy and programmed correctly to acknowledge receipt. + +Unfortunately, this isn't realistic. Things happen. Processes lock up. Networks go down. If we tell {pulsar-short} to indefinitely attempt message delivery to all subscribers on all topics, the backlog would grow out of control, with bookie disks continuously filling and never draining. So, guaranteed message delivery must be managed with some rules. + +{pulsar-short} has different ways of managing the broker's backlog (ie: guaranteed message delivery). Combining these different settings make up the rules of message retention. The rules of message retention directly impact how full a bookie's disk can be. We can't cover every possability within {pulsar-short}'s message retention system, so we will focus on 3 key areas and let those drive our sizing calculation. For more on message retention and expiration, see https://pulsar.apache.org/docs/concepts-messaging/#message-retention-and-expiry[{pulsar-short}'s message retention and expiry documentation]. + +=== Retention policy + +The broker's goal is to mark a message for deletion, which means all subscribers have acknowledged message receipt and the message can be removed from the bookie's disk. Don't confuse this with {pulsar-short}'s tiered storage, where you can store the broker's backlog for a very long time. This is a different concept than retention. Sometimes you want acknowledged messages to stay on disk for a certain period of time, or until a certain size threshold has been reached. For example, when a client is constantly reading a topic's messages and needs to have the same low latency performance as a consumer of unacknowledged messages, a highly performant reader is required. + +Retention can be expressed in size or time. Expressed as size, when the broker's backlog reaches some size threshold (in Mb), it will begin marking the oldest acknowledged messages for deletion until the size is reduced. Expressed in time, any acknowledged messages older than some time period (like 3 hours) will be marked for deletion. Size and time can also be used together to create a more comprehensive retention rule. + +{pulsar-short}'s default behavior disables retention policy, so our sizing calculations will assume this configuration. When all subscribers have acknowledged, the message is removed. + +=== Backlog quota size + +As mentioned above, the broker's backlog size is directly proportional to how much disk is being consumed on a bookie. {pulsar-short} provides the option to set thresholds of how large the backlog of a certain namespace can get. A policy can also be set to manage behavior for when that backlog threshold is passed. + +{pulsar-short}'s default is to not set a backlog quote on a namespace, so our sizing calculations will assume this configuration. + +=== Message time to live (TTL) + +TTL determines how long an unacknowledged message will last in the backlog before it is marked for deletion. {pulsar-short}'s default behavior disables TTL and stores unacked messages forever, but in a production cluster, there must be limits in place to prevent bookie disks from filling up and crippling a cluster's health. + +The TTL parameter is like a stopwatch attached to each message that defines the amount of time a message is allowed to stay unacknowledged. When the TTL expires, {pulsar-short} automatically moves the message to the acknowledged state (and thus makes it ready for deletion). + +TTL is expressed in terms of time, at the namespace level. A default value for all new namespace can be set with the `ttlDurationDefaultInSeconds` broker configuration value. + +== Aggregated cluster workload + +To size a cluster, you need a general understanding of what workloads it will be running. +Realistically, it's almost impossible to definitively know the exact applications and message sizes that will be used. If your cluster is successful, more teams will want to use it! So we've collected the "building blocks" of sizing a cluster, which we call an "aggregated cluster workload". Think of it as a loosely calculated algorithm to approximate cluster sizing. + +* _Average message size (uncompressed)_ - this is the most important number to understand. A message is sized by the number of bytes. A message includes its *message key*, *properties*, and a *message payload*. A *message key* is roughly the same number of characters as a GUID (or hash). *Message properties* is a key/value collection of metadata, so the number of characters varies. The *message payload* accounts for the bulk of the sizing variability. To start, assume the message is a JSON string with some number of characters. +For more on message compression, see the https://pulsar.apache.org/docs/concepts-messaging/#compression[{pulsar-short} documentation], or search for "calculate bytes of string" in your favorite search engine - you'll find many free tools where you can type out a sample JSON-formatted string and see the byte count. + +* _Incoming message throughput_ - this is the second most important number to understand. Throughput is expressed as a number of messages that the cluster can produce in a second. Think about this number in terms of steady traffic and burst traffic. {pulsar-short} can scale brokers to handle bursts, so you don't need to size for maximum workload, but you do need to consider the time it takes to scale up broker instances. If you were streaming in data every time someone clicked on a web page, and the site received a constant 2000 views per second, then your minimum throughput must be able to handle a load above that requirement, because that stream won't be the only load on the cluster. You likewise wouldn't size the cluster to your existing load, because you hope that load will grow over time. + +* _Message retention and TTL period_ - the size or time acknowledged messages are kept on disk. See message retention above for more detail. + +* _Tiered storage policies_ - Tiered storage offloads {bookkeeper-short} data to cheaper, long-term storage, and can impact cluster sizing if that storage service is included in the cluster. For our calculations we will not be including this feature. For more on tiered storage, see https://pulsar.apache.org/docs/tiered-storage-overview/[{pulsar-short} documentation]. + +There are other factors that could be a part of the aggregated cluster workload. As you gain familiarity with {pulsar-short} you can further customize this calculation. For now, we will estimate with the above numbers to size a cluster. + +[#aggregate-worksheet] +== Example workload aggregation worksheet + +Gather the following workload characteristics to determine your cluster's size requirements: + +.Workload input characteristics +[cols=2*,options=header] +|=== +|Workload input |Value + +| Average message size +| 1 Kb + +| Incoming message throughput +| 100000 messages per second + +| Message retention +| Disabled + +| TTL Policy +| 3 hours + +| Tiered storage +| N/A + +|=== + +== Example methodology + +With the aggregated workload characteristics, we can now apply our methodology to these characteristics to size a production cluster. + +First, size the {bookkeeper-short} disk because it's the most important component (bookies store message data) and the hardest to scale. +By default, {pulsar-short} sets {bookkeeper-short} https://pulsar.apache.org/docs/administration-zk-bk/#bookkeeper-persistence-policies[ack-quorum] size to 2. +That means at least 2 bookies in the ensemble need to acknowledge receipt of message data before {pulsar-short} will acknowledge receipt of the message. +But (very important) we want the message replication factor to be an odd number, so we can tolerate 1 bookie failure. + +. Multiply replication factor (3) by average message payload size (1) by average message throughput (100000), then factor in TTL (3) and retention period (3600) (when applicable). ++ +[source,plaintext] +---- +Total message size (raw) = +3 * // replication factor +1 Kb * // average message payload size +100000 * // average message throughput +(3 * 60 * 60) // TTL in seconds += 3,240,000,000 Kb +≅ 3 Tb +---- ++ +We now know our cluster needs 3 Tb of storage for {bookkeeper-short} ledger data. + +. Calculate the number of {bookkeeper-short} nodes with an individual ledger disk capacity. ++ +[source,plain,subs="+attributes"] +---- +{bookkeeper-short} count(raw)=ceiling(3/(4 * 0.85)) = 1 +---- ++ +If our bookie has a 4Tb disk and we anticipate at least 3Tb of workload, only 1 bookie is needed. +For fault tolerance, we adjust this to a number that is divisible by the number of zones, which equals 3 bookies. + +. Given the replication factor of 3, we will need at least 1 broker to write messages to the bookies. That gives us a broker-to-{bookkeeper-short} ratio of 1:3. Now we can calculate the total number of brokers across 3 zones. ++ +[source,plain] +---- +broker count(raw)=ceiling(1/3) = 1 +---- +We need 1 broker to serve messages. +As with other components, this must account for fault tolerance. +To be evenly divisible by the number of zones, we will set brokers to 3. + +=== {pulsar-short} component instance counts + +Now that we know how many server instances of broker and bookie are required to support our workload, we include the other components to size the overall cluster. + +.{pulsar-short} cluster component count +[cols="2,2,2", options=header] +|=== +|Component +|VM Count +|Notes + +|{zookeeper-short} +|3 +|1 per zone + +|{bookkeeper-short} (bookie) +|3 +|Calculated above + +|Broker +|3 +|Calculated above + +|Proxy +|3 +|1 per zone + +|AutoRecovery +|3 +|1 per zone + +|Function workers +|3 +|1 per zone + +|Admin +|1 +|1 per cluster + +|Heartbeat +|1 +|1 per cluster + +|=== + +== Next steps + +See xref:cluster-sizing-reference.adoc[] for specific deployment topologies and hardware recommendations. \ No newline at end of file diff --git a/modules/install-upgrade/pages/quickstart-helm-installs.adoc b/modules/install-upgrade/pages/quickstart-helm-installs.adoc index 05e66a2..4fa9d39 100644 --- a/modules/install-upgrade/pages/quickstart-helm-installs.adoc +++ b/modules/install-upgrade/pages/quickstart-helm-installs.adoc @@ -1,9 +1,14 @@ = Quick Start for Helm Chart installs -You have options for installing *{company} {product}*: +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +You have options for installing {product}: * With the provided *{company} Helm chart* for an existing Kubernetes environment locally or with a cloud provider, as covered in this topic. -* With the *{company} {product} tarball* for deployment to a single server/VM, or to multiple servers/VMs. See xref:install-upgrade:quickstart-server-installs.adoc[Quick Start for Server/VM installs]. +* With the *{product} tarball* for deployment to a single server/VM, or to multiple servers/VMs. See xref:install-upgrade:quickstart-server-installs.adoc[Quick Start for Server/VM installs]. * With the *{company} Ansible scripts* provided at {pulsar-ansible-repo}[{pulsar-ansible-repo}]. The Helm chart and options described below configure an {pulsar} cluster. diff --git a/modules/install-upgrade/pages/quickstart-server-installs.adoc b/modules/install-upgrade/pages/quickstart-server-installs.adoc index 083ed6e..7161e18 100644 --- a/modules/install-upgrade/pages/quickstart-server-installs.adoc +++ b/modules/install-upgrade/pages/quickstart-server-installs.adoc @@ -2,6 +2,11 @@ This document explains xref:install-upgrade:quickstart-server-installs.adoc#install[installation] of {product} for Bare Metal/VM deployments with a {pulsar-short} tarball. +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + The resulting {product} deployment includes: * *Tiered Storage:* Offload historical messages to more cost effective object storages such as AWS S3, Azure Blob, Google Cloud Storage, and HDFS. @@ -43,7 +48,7 @@ Check this setting with `cat /sys/kernel/mm/transparent_hugepage/enabled` and `c [#install] == Installation -. Download the {company} {product} tarball from the {pulsar-repo}/releases[{company} GitHub repo]. There are three versions of {product} currently available: +. Download the {product} tarball from the {pulsar-repo}/releases[{company} GitHub repo]. There are three versions of {product} currently available: + [cols="1,1"] [%autowidth] @@ -89,11 +94,11 @@ drwxr-xr-x@ 277 firstname.lastname staff 8864 May 17 05:58 lib drwxr-xr-x@ 25 firstname.lastname staff 800 Jan 22 2020 licenses ---- -You have successfully installed the {company} {product} tarball. +You have successfully installed the {product} tarball. == Additional tooling -Once the {company} {product} tarball is installed, you may want to add additional tooling to your server/VM deployment. +Once the {product} tarball is installed, you may want to add additional tooling to your server/VM deployment. * *{pulsar-short} Admin Console:* Web-based UI that administrates {pulsar-short}. Download the latest version from the {pulsar-admin-console-repo}[{company} GitHub repo] and follow the instructions xref:components:admin-console-vm.adoc[here]. diff --git a/modules/install-upgrade/pages/supported-versions.adoc b/modules/install-upgrade/pages/supported-versions.adoc new file mode 100644 index 0000000..55f6241 --- /dev/null +++ b/modules/install-upgrade/pages/supported-versions.adoc @@ -0,0 +1,94 @@ += Supported software +:page-aliases: 2.10-1.x@install-upgrade:supported-versions.adoc, 2.10-2.x@install-upgrade:supported-versions.adoc + +Support covers only the following Software versions for {pulsar}: + +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +[cols="2*"] +|=== +|Name |Versions + +|{pulsar} +|3.1.x +|=== + +Support covers only the following Software versions for {product} Distribution: + +[cols="2*"] +|=== +|Name |Version + +|{product} Distribution +|v.1.0.0 and above + +|=== + +Support covers only the following Software versions for the Open Source Projects: + +[cols="2*"] +|=== +|Name |Version + +|{pulsar-short} Admin Console +|v.1.0.0 and above + +|{pulsar-short} Heartbeat +|v.1.0.0 and above + +|{company} pulsar-sink +|v.1.0.0 and above + +|Starlight for JMS +|v.1.0.0 and above + +|{starlight-rabbitmq} +|v.1.0.0 and above + +|=== + +Support covers only the following {product} Connectors and {pulsar} Connectors (as included in the {pulsar} download): + +[cols="2*"] +|=== +|Name |Version + +|{cass-short} enhanced sink +|v1.6.11 +|Cloud storage sink +|v2.10.0 +|Elasticsearch sink +|v3.1.x +|HTTP sink +|v3.1.x +|JDBC-Clickhouse sink +|v3.1.x +|JDBC-MariaDB sink +|v3.1.x +|JDBC-Postgres sink +|v3.1.x +|JDBC-SQLite +|v3.1.x +|Kinesis sink +|v3.1.x +|Snowflake sink +|v0.1.13 +|{cass-short} source +|v2.2.9 +|Data generator source +|v3.1.x +|Debezium MongoDB source +|v3.1.x +|Debezium MySQL source +|v3.1.x +|Debezium Microsoft SQL Server Source +|v3.1.x +|Debezium Postgres source +|v3.1.x +|Kinesis source +|v3.1.x + +|=== \ No newline at end of file diff --git a/modules/install-upgrade/pages/upgrade-guide.adoc b/modules/install-upgrade/pages/upgrade-guide.adoc new file mode 100644 index 0000000..ff2e034 --- /dev/null +++ b/modules/install-upgrade/pages/upgrade-guide.adoc @@ -0,0 +1,454 @@ += Upgrade {product} from 2.10 to 3.1 +:navtitle: Upgrade from 2.10 to 3.1 + +This guide provides instructions and recommendations for upgrading {product} from version 2.10 to 3.1. + +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +Upgrades to {product} 3.1 should only be performed from {product} 2.10. + +All {product} 2.10 components support the upgrade to 3.1. + +== Functional impacts + +This section describes changes in {product} 3.1 that may impact how your deployment functions. + +=== Default system topics + +In {pulsar-short} 3.1, system topics are now enabled by default. + +=== Prometheus metrics changes + +Prometheus metrics have been updated in {product} 3.1. + +* Prometheus Client version has changed from `0.5.0` to `0.16.0` +* Prometheus Metric type `UNTYPED` is renamed to `UNKNOWN` +* Metrics have been renamed because OpenMetrics's counter name needs a `_total` suffix + +.Renamed metrics +[cols="2,2"] +|=== +|{product} 2.10 |{product} 3.1 + +|pulsar_expired_token_count +|pulsar_expired_token_total + +|pulsar_authentication_success_count +|pulsar_authentication_success_total + +|pulsar_authentication_failures_count +|pulsar_authentication_failures_total + +|pulsar_source_received_total_1min +|pulsar_source_received_1min_total + +|pulsar_source_written_total_1min +|pulsar_source_written_1min_total + +|pulsar_source_source_total_1min +|pulsar_source_source_exceptions_1min_total + +|pulsar_source_system_exceptions_total_1min +|pulsar_source_system_exceptions_1min_total + +|pulsar_function_received_total_1min +|pulsar_function_received_1min_total + +|pulsar_function_user_exceptions_total_1min +|pulsar_function_user_exceptions_1min_total + +|pulsar_function_system_exceptions_total_1min +|pulsar_function_system_exceptions_1min_total + +|pulsar_function_processed_successfully_total_1min +|pulsar_function_processed_successfully_1min_total + +|pulsar_sink_received_total_1min +|pulsar_sink_received_1min_total + +|pulsar_sink_written_total_1min +|pulsar_sink_written_1min_total + +|pulsar_sink_sink_exceptions_total_1min +|pulsar_sink_sink_exceptions_1min_total + +|pulsar_sink_system_exceptions_total_1min +|pulsar_sink_system_exceptions_1min_total + +|pulsar_lb_unload_broker_count +|pulsar_lb_unload_broker_total + +|pulsar_lb_unload_bundle_count +|pulsar_lb_unload_bundle_total + +|pulsar_lb_bundles_split_count +|pulsar_lb_bundles_split_total + +|pulsar_schema_del_ops_failed_count +|pulsar_schema_del_ops_failed_total + +|pulsar_schema_get_ops_failed_count +|pulsar_schema_get_ops_failed_total + +|pulsar_schema_put_ops_failed_count +|pulsar_schema_put_ops_failed_total + +|pulsar_schema_compatible_count +|pulsar_schema_compatible_total + +|pulsar_schema_incompatible_count +|pulsar_schema_incompatible_total + +|pulsar_txn_committed_count +|pulsar_txn_committed_total + +|pulsar_txn_aborted_count +|pulsar_txn_aborted_total + +|pulsar_txn_created_count +|pulsar_txn_created_total + +|pulsar_txn_timeout_count +|pulsar_txn_timeout_total + +|pulsar_txn_append_log_count +|pulsar_txn_append_log_total +|=== + +The following PRs were merged to update metrics: + +* {apache-pulsar-repo}/pull/13785[#13785 - Bump prometheus client version from 0.5.0 to 0.15.0] +* {apache-pulsar-repo}/pull/16581[#16581 - Rename {pulsar-short} txn metrics to specify OpenMetrics] +* {apache-pulsar-repo}/pull/16610[#16610 - Rename {pulsar-short} schema metrics to specify OpenMetrics] +* {apache-pulsar-repo}/pull/16611[#16611 - Rename {pulsar-short} lb metrics to specify OpenMetrics] +* {apache-pulsar-repo}/pull/16591[#16591 - Bump prometheus client version from 0.15.0 to 0.16.0] +* {apache-pulsar-repo}/pull/17419[#17419 - Removed timestamp from all prometheus metrics.] + + +=== Other functional impacts + +The following PRs were merged in {product} 3.1 that may impact your deployment's functionality. + +[cols="1,2,3"] +|=== +|PR Link |Title |Functional Impact + +|{apache-pulsar-repo}/pull/19180[#19180] +|Deprecate blocking AuthorizationService, AuthorizationProvider methods +|This will affect the public API for the AuthorizationService and the AuthorizationProvider, which only impacts users that are running custom code inside the {pulsar-short} broker + +|{apache-pulsar-repo}/pull/19182[#19182] +|Remove AuthorizationProvider methods deprecated in 2.7 and 2.9 +|Removing deprecated methods allowTenantOperationAsync, allowTenantOperation, allowNamespaceOperationAsync, allowNamespaceOperation, allowNamespacePolicyOperationAsync, allowNamespacePolicyOperation, allowTopicOperationAsync, allowTopicOperation. These methods could be used by third party extensions + +|{apache-pulsar-repo}/pull/19197[#19197] +|Update AuthenticationProvider to simplify HTTP Authn +|This changes the public API within the broker as some methods are marked as @Deprecated + +|{apache-pulsar-repo}/pull/19295[#19295] +|OneStageAuth State: move authn out of constructor +|This could break 3rd party plugins in the broker if they were relying on authentication to happen in the constructor. In order to make those implementations fail fast, this PR includes a change to throw an exception when the getAuthRole is called without first calling authenticateAsync or authenticate. That makes these changes semi-backwards compatible. + +|{apache-pulsar-repo}/pull/19314[#19314] +|TokenAuthenticationState: authenticate token only once +|In a sense, this breaks an implicit contract that the class had. However, because the getAuthRole() method will throw an exception if called incorrectly, it is likely that misuse of this class will result in a fail fast behavior. + +|{apache-pulsar-repo}/pull/19455[#19455] +|Require authRole is proxyRole to set originalPrincipal +|This change affects the binary protocol's usage without changing the binary protocol itself. Upgrading existing proxies will not work if the proxyRoles is not correctly configured in the broker.conf. + +|{apache-pulsar-repo}/pull/19486[#19486] +|Remove default 30s ackTimeout when setting DLQ policy on java consumer +|Removed setting default ackTimeoutMillis in java ConsumerBuilder when a deadLetterPolicy is set. It has to be specified exclusively to use. +|=== + +== Configuration impacts + +This section describes changes in {product} 3.1 that may impact your deployment's configuration. + +=== Configuration values removed in 3.1 + +* {apache-pulsar-repo}/pull/14506[PR #14506] removes `managedLedgerNumWorkerThreads`. +The `MetadataStore` instance is now passed from the `PulsarService` directly to the `ManagedLedgerFactory`. + +* The {pulsar-short} SQL `conf/presto` directory has been removed. +** If you're upgrading {pulsar-short} SQL from 2.11 or earlier, copy the {pulsar-short} SQL config files from `conf/presto` to `trino/conf`. +** If you're downgrading {pulsar-short} SQL to 2.11 or earlier from newer versions, copy the {pulsar-short} SQL config files from `trino/conf` to `conf/presto`. + +=== Default values changed or deprecated in 3.1 + +The following default values in `broker.conf` and `standalone.conf` have changed or been deprecated in {product} 3.1. + +.`broker.conf` and `standalone.conf` values +[cols="1,1,1"] +|=== +|Configuration |{product} 2.10 Default | {product} 3.1 Default + +|`managedLedgerCacheEvictionFrequency`` +|`100.0` +|`0` + +|`managedLedgerMaxUnackedRangesToPersistInZooKeeper` +|`1000` +|`-1` + +|`systemTopicEnabled` +|`false` +|`true` + +|`topicLevelPoliciesEnabled` +|`false` +|`true` + +|`supportedNamespaceBundleSplitAlgorithms` +|`range_equally_divide`,`topic_count_equally_divide`,`specified_positions_divide` +|`range_equally_divide`,`topic_count_equally_divide`,`specified_positions_divide`,`flow_or_qps_equally_divide` + +|`loadBalancerDirectMemoryResourceWeight` +|`1.0` +|`0` + +|`fileSystemProfilePath` +|`../conf/filesystem_offload_core_site.xml` +|`conf/filesystem_offload_core_site.xml` + +|`gcsManagedLedgerOffloadMaxBlockSizeInBytes` +|`67108864` +|`134217728` +|=== + +== Operational impacts + +This section describes changes in {product} 3.1 that may impact how your deployment operates. + +=== JDK 17 upgrade + +{product} 3.1 uses JDK 17. + +The {pulsar-short} server module's `javac` release version is `17`. + +Client and client-server shared modules remain at the target Java 8 release. + +This modification is described in detail in {apache-pulsar-repo}/pull/15207[PIP-156]. + +=== Python 2 support removed + +{product} 3.1 removes Python 2 from build scripts. + +Python 3 is used in the build image. + +The build image is updated to use `ubuntu:20.04`, as there is no Python 3.7 support in the previous Ubuntu image. + +Executable scripts have been updated to invoke `python3` instead of `python`. + +This modification is described in detail in {apache-pulsar-repo}/pull/15376[PIP-155] + +== Known issues + +This section describes known issues encountered when upgrading to {product} 3.1. + +=== {bookkeeper-short} and RocksDB format + +[IMPORTANT] +==== +Downgrading to {product} 2.10 from {product} 3.1 is not supported for the {bookkeeper-reg} and {zookeeper-reg} components. +==== + +{pulsar-short} 3.1 uses RocksDB `7.x`, which writes in a format that is not compatible with RocksDB `6.x`. + +{product} 2.10 uses {bookkeeper} 4.14, which uses RocksDB `6.x`. + +All other components such as broker, proxy, and Functions Worker can be downgraded at any time. + +For more information, see {apache-pulsar-repo}/issues/22051[Issue 22051]. + +== Upgrade procedure + +{product} can be deployed on bare metal, Docker, and Kubernetes. + +This guide only addresses Kubernetes deployment. + +For more information on upgrading bare metal and Docker {pulsar-short} deployments, see the https://pulsar.apache.org/docs/3.3.x/administration-upgrade/[{pulsar-short} documentation]. + +=== Upgrade Kubernetes deployment with {kaap-short} + +Upgrade to {product} 3.1 on Kubernetes with {kaap}. + +For more information, see the xref:kaap-operator::index.adoc[{kaap-short} documentation]. + +. To prevent data loss, back up your existing {pulsar-short} data and configuration files. +. To save your current Helm release configuration, run the following command: ++ +[source,bash,subs="+quotes"] +---- +helm get values *RELEASE-NAME* > pulsar-backup-values.yaml +---- ++ +. To update the {product} Helm chart repository, run the following command: ++ +[source,bash] +---- +helm repo update +---- ++ +. Open `helm/kaap-stack/values.yaml`, and then update the image tag to `3.1.0` (or the specific tag you wish to use). ++ +[source,yaml] +---- +kaap: + enabled: true + cluster: + name: pulsar + create: true + spec: + global: + name: pulsar + image: + datastax/lunastreaming-all: 3.1_4.5 +---- ++ +. To modify other configurations, update `values.yaml` as needed. For example, to modify the broker's namespace shedding and splitting configurations, update the following fields: ++ +[source,yaml] +---- +kaap: + enabled: true + cluster: + name: pulsar + create: true + spec: + global: + name: pulsar + broker: + replicas: 2 + config: + loadBalancerNamespaceBundleSplitConditionHitCountThreshold: 1 + loadBalancerSheddingConditionHitCountThreshold: 1e +---- ++ +. To upgrade your existing {pulsar-short} installation, run the following Helm command. +This command assumes the default `pulsar` namespace. If you are using a different namespace, replace `pulsar` with your namespace. +The `--wait` flag ensures that Helm waits until all pods are ready before completing the upgrade. ++ +[source,bash,subs="+quotes"] +---- +helm upgrade --namespace *NAMESPACE* --wait --debug --timeout 1200s \ +--dependency-update pulsar *KAAP-REPO-DIRECTORY*/helm/kaap-stack \ +--values *PATH-TO-CURRENT-VALUES-FILE*.yaml +---- ++ +. Check the status of the pods to ensure they are running correctly: ++ +[source,bash,subs="+quotes"] +---- +kubectl get pods --namespace *NAMESPACE* +---- + +. Check the logs for any issues: ++ +[source,bash,subs="+quotes"] +---- +kubectl logs *POD-NAME* -n *NAMESPACE* +---- + +. After the upgrade, ensure all necessary configurations are in place and correct. + +=== Upgrade Kubernetes deployment with Helm chart + +The Helm chart for {product} is available in the {pulsar-helm-chart-repo}/blob/master/helm-chart-sources/pulsar/values.yaml[Helm chart sources] repository. + +. To prevent data loss, back up your existing {pulsar-short} data and configuration files. +. To save your current Helm release configuration, run the following command: ++ +[source,bash,subs="+quotes"] +---- +helm get values *RELEASE-NAME* > pulsar-backup-values.yaml +---- ++ +. To update the {product} Helm chart repository, run the following command: ++ +[source,bash] +---- +helm repo update +---- ++ +. Open `helm-chart-sources/pulsar/values.yaml` and update the image tag to `3.1.0` (or the specific tag you wish to use). ++ +[source,yaml] +---- +image: + broker: + # If not using tiered storage, you can use the smaller pulsar image + # for the broker + repository: datastax/lunastreaming-all + pullPolicy: IfNotPresent + tag: 3.1_4.5 + brokerSts: + # If not using tiered storage, you can use the smaller pulsar image + # for the broker + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latest + function: + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latest + zookeeper: + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latestupgr + bookkeeper: + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latest + proxy: + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latest + bastion: + repository: apachepulsar/pulsar + pullPolicy: IfNotPresent + tag: latest +---- +. Review and modify any other configuration parameters that may have changed between versions, such as resource limits, storage classes, and additional components. To modify other configurations, update `values.yaml` as needed. For example, to modify the broker's replica count, update the following fields: ++ +[source,yaml] +---- +broker: + component: broker + replicaCount: 2 + configData: + brokerDeduplicationEnabled: "false" +---- + +. To upgrade your existing {pulsar-short} installation, run the following Helm command. +This command assumes the default `pulsar` namespace. If you are using a different namespace, replace `pulsar` with your namespace. +The `--wait` flag ensures that Helm waits until all pods are ready before completing the upgrade. ++ +[source,bash,subs="+quotes"] +---- +helm upgrade --namespace *NAMESPACE* --wait --debug --timeout 1200s \ +--dependency-update pulsar *KAAP-REPO-DIRECTORY*/helm/kaap-stack \ +--values *PATH-TO-CURRENT-VALUES-FILE*.yaml +---- + +. To check the status of the pods to ensure they are running correctly, run the following command: ++ +[source,bash,subs="+quotes"] +---- +kubectl get pods --namespace *NAMESPACE* +---- + +. To check the logs for any issues, run the following command: ++ +[source,bash,subs="+quotes"] +---- +kubectl logs *POD-NAME* -n *NAMESPACE* +---- + +. After the upgrade, ensure all necessary configurations are in place and correct. + + diff --git a/modules/operations/pages/functions-transform.adoc b/modules/operations/pages/functions-transform.adoc index e86dedb..0e964a7 100644 --- a/modules/operations/pages/functions-transform.adoc +++ b/modules/operations/pages/functions-transform.adoc @@ -1,4 +1,4 @@ -= {product} {pulsar-short} transform functions configuration reference += {pulsar-short} transform functions configuration reference :navtitle: Transform functions configuration reference include::common:streaming:partial$transform-functions/intro.adoc[] diff --git a/modules/operations/pages/functions.adoc b/modules/operations/pages/functions.adoc index 207bc49..c2afee3 100644 --- a/modules/operations/pages/functions.adoc +++ b/modules/operations/pages/functions.adoc @@ -1,10 +1,16 @@ -= Deploy and manage {product} {pulsar-short} functions += Deploy and manage {pulsar-short} functions :navtitle: Deploy and manage {pulsar-short} functions Functions are lightweight compute processes that you can run on each message a topic receives. You can apply custom logic to a message, transforming or enriching it, and then output it to a different topic. Functions run inside {product}, which makes them serverless. + +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + You write the code for your function in Java, Python, or Go, and then upload the code to the {pulsar-short} cluster and deploy the function. The function automatically runs for each message published to the specified input topic. diff --git a/modules/operations/pages/scale-cluster.adoc b/modules/operations/pages/scale-cluster.adoc index 58c2067..07653e5 100644 --- a/modules/operations/pages/scale-cluster.adoc +++ b/modules/operations/pages/scale-cluster.adoc @@ -2,7 +2,12 @@ This page will show you how to scale {product} clusters up for more compute capacity, or down for less. -== Installing {pulsar-short} cluster +[NOTE] +==== +include::ROOT:partial$luna-name.adoc[] +==== + +include::operations:partial$operator-scaling.adoc[] For our {pulsar-short} cluster installation, use this {pulsar-helm-chart-repo}[Helm chart]. From ba55caa16937dda1cb7070f263452d47801b9d06 Mon Sep 17 00:00:00 2001 From: Mendon Kissling <59585235+mendonk@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:02:27 -0400 Subject: [PATCH 3/5] not-in-release --- .../pages/production-cluster-sizing.adoc | 238 --------- .../pages/supported-versions.adoc | 94 ---- .../install-upgrade/pages/upgrade-guide.adoc | 454 ------------------ 3 files changed, 786 deletions(-) delete mode 100644 modules/install-upgrade/pages/production-cluster-sizing.adoc delete mode 100644 modules/install-upgrade/pages/supported-versions.adoc delete mode 100644 modules/install-upgrade/pages/upgrade-guide.adoc diff --git a/modules/install-upgrade/pages/production-cluster-sizing.adoc b/modules/install-upgrade/pages/production-cluster-sizing.adoc deleted file mode 100644 index 1ffad6a..0000000 --- a/modules/install-upgrade/pages/production-cluster-sizing.adoc +++ /dev/null @@ -1,238 +0,0 @@ -= Production Cluster Sizing - -This document summarizes {company}'s recommendations for the sizing and optimization of an {pulsar} cluster on Linux in a production environment. -Remember, a {pulsar-short} *instance* is made of one or many clusters. - -Of course, the sizing of a cluster depends on factors like use case and expected load, so this document is not intended to be a one-size-fits-all guide. Rather, we'd like to demonstrate how we consider and spec the initial size of a {pulsar-short} cluster, and assist you on your journey to unlocking the scaling power of {pulsar-short}. - -This page summarizes the requirements, assumptions, definitions, and methodologies that inform our cluster sizing recommendations. -If you're looking for specific deployment topology recommendations, see xref:cluster-sizing-reference.adoc[]. - -include::operations:partial$operator-scaling.adoc[] - -== Dedicated VMs or Kubernetes - -As you begin your journey to design an {pulsar} cluster, one of the first questions to consider is what infrastructure your cluster will run on. -Most of this guide will focus on running a cluster with dedicated virtual machines. -While Kubernetes is the more popular option, it is easier to express disk calculations, throughput, and secure communications in terms of a VM. - -== {pulsar-short} cluster components - -{pulsar-short} clusters come in many shapes and sizes. There are minimum components for base functionality, and there are recommended components that make message routing, management, and observability easier. For this guide we will focus on the required components and what it takes to make them resilient to outages and highly available in a three-zone cloud. - -=== Required components - -* https://pulsar.apache.org/docs/concepts-architecture-overview/#metadata-store[{zookeeper-reg}] - This is {pulsar-short}'s meta data store. It stores data about a cluster's configuration, helps the proxy direct messages to the correct broker, and holds bookie configurations. Start with 1 instance of {zookeeper-short} in each availability zone (AZ) to mitigate a single failure point, and scale {zookeeper-short} as cluster traffic increases. You could scale {zookeeper-short} as traffic within the cluster increases, but it shouldn't be very often as it can handle quite a bit of load. - -* https://pulsar.apache.org/docs/concepts-architecture-overview/#brokers[Broker] - This is {pulsar-short}'s message router. -Ideally, each broker should be fully utilized without becoming a performance bottleneck. -The {pulsar-short} broker is stateless, so it requires considerable computing power but not much storage. -Start with 1 broker instance in each zone, and set a scaling rule that watches CPU load. -The best way to optimize this is through performance testing based on your cluster's workload characteristics. - -* https://pulsar.apache.org/docs/concepts-architecture-overview/#apache-bookkeeper[{bookkeeper-reg} (bookie)] - This is {pulsar-short}'s data store. -{bookkeeper-short} stores message data in a low-latency, resilient way. -{pulsar-short} uses {bookkeeper-short}'s quorum math to function, so a loss of 1 {bookkeeper-short} instance won't bring your system down, but will cause some data loss. -Start with at least 3 bookies, with 1 in each AZ. -At least 2 bookies per AZ are required for high availability, so if one bookie goes down, the other bookie in the AZ can take over. -Scale bookies up on disc usage percentage. Scale down manually by making a bookie read-only, offloading its data, then terminating the instance. - -[#recommended] -=== Recommended server components - -The {product} Helm chart deployment includes optional but highly recommended server components for better {pulsar-short} cluster metrics monitoring and operation visibility. - -* https://bookkeeper.apache.org/docs/admin/autorecovery[{bookkeeper-short} AutoRecovery] - This is a {pulsar-short} component that recovers {bookkeeper-short} data in the event of a bookie outage. While optional you will want the insurance of AutoRecovery working on your behalf. -A single instance of AutoRecovery should be adequate - only in the most heavily-used clusters will you need more. -* https://pulsar.apache.org/docs/concepts-architecture-overview/#pulsar-proxy[{pulsar-short} proxy] - The {pulsar-short} proxy is just that - a proxy. -It runs at the edge of the cluster with public facing endpoints. -Without it, your brokers would expose those endpoints, which is not an ideal configuration in production. -{pulsar-short} proxy also offers special options for cluster extensions, like the xref:components:starlight.adoc[{company} Starlight suite of APIs]. -Start with a proxy in each zone. -The proxy will be made aware of all the brokers in the same zone and load balance across them. -Have your load balancer round-robin to all proxy instances in all zones. -Proxy is optional for VM deployments and required for Kubernetes deployments. -Scale proxies by their network load or (if running extensions) also scale on CPU usage. -* https://pulsar.apache.org/docs/functions-worker-run-separately/[Dedicated functions worker(s)] - You can optionally run dedicated function workers in a {pulsar-short} cluster. -Without dedicated function workers, functions run as a separate process on the broker. -Function worker spec is usually focused on compute and memory. -Scale the workers based on overall usage (both CPU and memory). -* xref:luna-streaming:components:admin-console-tutorial.adoc[{pulsar-short} AdminConsole] - This is an optional web-based admin console for managing {pulsar-short} clusters, and makes management much easier than tons of CLI commands. The sizing and scaling for AdminConsole has nothing to do with the cluster, as it is not a failure point. -* xref:luna-streaming:components:heartbeat-vm.adoc[{pulsar-short} Heartbeat] - This is an optional component that monitors the health of {pulsar-short} cluster and emits metrics about the cluster that are helpful for observing and debugging issues. -* Prometheus/Grafana/Alert manager stack - This is the default observability stack for a cluster. The Luna Helm chart includes pre-made dashboards in Grafana and pre-wires all the metrics scraping. - -image::pulsar-components.png[] - -[#message-retention] -== Message retention - -The broker ensures messages are received and delivered appropriately, but it is a stateless process so it doesn't use its memory to track this. Instead, the broker uses {bookkeeper-short}s (bookies) to store message data and the message's acknowledgement state. -A great benefit of {bookkeeper-short} is its quorum policies. These policies make each bookie aware of the other bookies to form a {bookkeeper-short} cluster. With a cluster established, the cluster can have acknowledgement rules that form a data replication factor. For example, if you had 3 bookies in a {bookkeeper-short} cluster with an acknowledgement rule that at least 2 of the 3 bookies must have a copy of the data, then the cluster has a replication factor of 2. A {pulsar-short} broker uses the `managedLedgerDefaultAckQuorum` and `managedLedgerDefaultWriteQuorum` configurations to set the bounds of this rule. For more about {bookkeeper-short} persistence, see https://pulsar.apache.org/docs/administration-zk-bk/#bookkeeper-persistence-policies[here]. - -When a client produces a message, the broker will not acknowledge receipt until the replication factor has been achieved. Continuing from the above example, if the replication factor is 2, a broker's acknowledgment means a minimum of 2 bookies have confirmed storage of message data. If the broker times out waiting for at least 2 responses from the bookies, then the broker will not acknowledge receipt with the client. The client will need to handle the exception by attempting to resend or fail. This process forms one of {pulsar-short}'s core values - guaranteed message receipt. - -Now that the broker has a message, it guarantees delivery to the associated topic's subscribers. We refer to this as the broker's backlog. The size of the backlog is sometimes expressed by the number of messages. For example, a {pulsar-short} operator might say, "we try to keep the backlog below 100 messages." The number of brokers available to process messages directly impacts the size of the backlog. However, the number of messages is not a meaningful number on its own without knowing the size of the messages. Message size is essential information because it determines how full a bookie's disk will be. If the backlog has 100 messages that are 4Gb each, then 400Gb is occupied on a bookie's disk. If the backlog has 100 messages that are 1Kb each, then only 100Kb is occupied on the bookie's disk. Quite a difference in storage capacity! - -Until all subscribers have acknowledged receipt of a message, the broker will not mark a message as acknowledged. This is another core feature of {pulsar-short} - guaranteed message delivery. But there are realities around this - we must assume that all functions, sinks, and clients subscribed to the message's topic are healthy and programmed correctly to acknowledge receipt. - -Unfortunately, this isn't realistic. Things happen. Processes lock up. Networks go down. If we tell {pulsar-short} to indefinitely attempt message delivery to all subscribers on all topics, the backlog would grow out of control, with bookie disks continuously filling and never draining. So, guaranteed message delivery must be managed with some rules. - -{pulsar-short} has different ways of managing the broker's backlog (ie: guaranteed message delivery). Combining these different settings make up the rules of message retention. The rules of message retention directly impact how full a bookie's disk can be. We can't cover every possability within {pulsar-short}'s message retention system, so we will focus on 3 key areas and let those drive our sizing calculation. For more on message retention and expiration, see https://pulsar.apache.org/docs/concepts-messaging/#message-retention-and-expiry[{pulsar-short}'s message retention and expiry documentation]. - -=== Retention policy - -The broker's goal is to mark a message for deletion, which means all subscribers have acknowledged message receipt and the message can be removed from the bookie's disk. Don't confuse this with {pulsar-short}'s tiered storage, where you can store the broker's backlog for a very long time. This is a different concept than retention. Sometimes you want acknowledged messages to stay on disk for a certain period of time, or until a certain size threshold has been reached. For example, when a client is constantly reading a topic's messages and needs to have the same low latency performance as a consumer of unacknowledged messages, a highly performant reader is required. - -Retention can be expressed in size or time. Expressed as size, when the broker's backlog reaches some size threshold (in Mb), it will begin marking the oldest acknowledged messages for deletion until the size is reduced. Expressed in time, any acknowledged messages older than some time period (like 3 hours) will be marked for deletion. Size and time can also be used together to create a more comprehensive retention rule. - -{pulsar-short}'s default behavior disables retention policy, so our sizing calculations will assume this configuration. When all subscribers have acknowledged, the message is removed. - -=== Backlog quota size - -As mentioned above, the broker's backlog size is directly proportional to how much disk is being consumed on a bookie. {pulsar-short} provides the option to set thresholds of how large the backlog of a certain namespace can get. A policy can also be set to manage behavior for when that backlog threshold is passed. - -{pulsar-short}'s default is to not set a backlog quote on a namespace, so our sizing calculations will assume this configuration. - -=== Message time to live (TTL) - -TTL determines how long an unacknowledged message will last in the backlog before it is marked for deletion. {pulsar-short}'s default behavior disables TTL and stores unacked messages forever, but in a production cluster, there must be limits in place to prevent bookie disks from filling up and crippling a cluster's health. - -The TTL parameter is like a stopwatch attached to each message that defines the amount of time a message is allowed to stay unacknowledged. When the TTL expires, {pulsar-short} automatically moves the message to the acknowledged state (and thus makes it ready for deletion). - -TTL is expressed in terms of time, at the namespace level. A default value for all new namespace can be set with the `ttlDurationDefaultInSeconds` broker configuration value. - -== Aggregated cluster workload - -To size a cluster, you need a general understanding of what workloads it will be running. -Realistically, it's almost impossible to definitively know the exact applications and message sizes that will be used. If your cluster is successful, more teams will want to use it! So we've collected the "building blocks" of sizing a cluster, which we call an "aggregated cluster workload". Think of it as a loosely calculated algorithm to approximate cluster sizing. - -* _Average message size (uncompressed)_ - this is the most important number to understand. A message is sized by the number of bytes. A message includes its *message key*, *properties*, and a *message payload*. A *message key* is roughly the same number of characters as a GUID (or hash). *Message properties* is a key/value collection of metadata, so the number of characters varies. The *message payload* accounts for the bulk of the sizing variability. To start, assume the message is a JSON string with some number of characters. -For more on message compression, see the https://pulsar.apache.org/docs/concepts-messaging/#compression[{pulsar-short} documentation], or search for "calculate bytes of string" in your favorite search engine - you'll find many free tools where you can type out a sample JSON-formatted string and see the byte count. - -* _Incoming message throughput_ - this is the second most important number to understand. Throughput is expressed as a number of messages that the cluster can produce in a second. Think about this number in terms of steady traffic and burst traffic. {pulsar-short} can scale brokers to handle bursts, so you don't need to size for maximum workload, but you do need to consider the time it takes to scale up broker instances. If you were streaming in data every time someone clicked on a web page, and the site received a constant 2000 views per second, then your minimum throughput must be able to handle a load above that requirement, because that stream won't be the only load on the cluster. You likewise wouldn't size the cluster to your existing load, because you hope that load will grow over time. - -* _Message retention and TTL period_ - the size or time acknowledged messages are kept on disk. See message retention above for more detail. - -* _Tiered storage policies_ - Tiered storage offloads {bookkeeper-short} data to cheaper, long-term storage, and can impact cluster sizing if that storage service is included in the cluster. For our calculations we will not be including this feature. For more on tiered storage, see https://pulsar.apache.org/docs/tiered-storage-overview/[{pulsar-short} documentation]. - -There are other factors that could be a part of the aggregated cluster workload. As you gain familiarity with {pulsar-short} you can further customize this calculation. For now, we will estimate with the above numbers to size a cluster. - -[#aggregate-worksheet] -== Example workload aggregation worksheet - -Gather the following workload characteristics to determine your cluster's size requirements: - -.Workload input characteristics -[cols=2*,options=header] -|=== -|Workload input |Value - -| Average message size -| 1 Kb - -| Incoming message throughput -| 100000 messages per second - -| Message retention -| Disabled - -| TTL Policy -| 3 hours - -| Tiered storage -| N/A - -|=== - -== Example methodology - -With the aggregated workload characteristics, we can now apply our methodology to these characteristics to size a production cluster. - -First, size the {bookkeeper-short} disk because it's the most important component (bookies store message data) and the hardest to scale. -By default, {pulsar-short} sets {bookkeeper-short} https://pulsar.apache.org/docs/administration-zk-bk/#bookkeeper-persistence-policies[ack-quorum] size to 2. -That means at least 2 bookies in the ensemble need to acknowledge receipt of message data before {pulsar-short} will acknowledge receipt of the message. -But (very important) we want the message replication factor to be an odd number, so we can tolerate 1 bookie failure. - -. Multiply replication factor (3) by average message payload size (1) by average message throughput (100000), then factor in TTL (3) and retention period (3600) (when applicable). -+ -[source,plaintext] ----- -Total message size (raw) = -3 * // replication factor -1 Kb * // average message payload size -100000 * // average message throughput -(3 * 60 * 60) // TTL in seconds -= 3,240,000,000 Kb -≅ 3 Tb ----- -+ -We now know our cluster needs 3 Tb of storage for {bookkeeper-short} ledger data. - -. Calculate the number of {bookkeeper-short} nodes with an individual ledger disk capacity. -+ -[source,plain,subs="+attributes"] ----- -{bookkeeper-short} count(raw)=ceiling(3/(4 * 0.85)) = 1 ----- -+ -If our bookie has a 4Tb disk and we anticipate at least 3Tb of workload, only 1 bookie is needed. -For fault tolerance, we adjust this to a number that is divisible by the number of zones, which equals 3 bookies. - -. Given the replication factor of 3, we will need at least 1 broker to write messages to the bookies. That gives us a broker-to-{bookkeeper-short} ratio of 1:3. Now we can calculate the total number of brokers across 3 zones. -+ -[source,plain] ----- -broker count(raw)=ceiling(1/3) = 1 ----- -We need 1 broker to serve messages. -As with other components, this must account for fault tolerance. -To be evenly divisible by the number of zones, we will set brokers to 3. - -=== {pulsar-short} component instance counts - -Now that we know how many server instances of broker and bookie are required to support our workload, we include the other components to size the overall cluster. - -.{pulsar-short} cluster component count -[cols="2,2,2", options=header] -|=== -|Component -|VM Count -|Notes - -|{zookeeper-short} -|3 -|1 per zone - -|{bookkeeper-short} (bookie) -|3 -|Calculated above - -|Broker -|3 -|Calculated above - -|Proxy -|3 -|1 per zone - -|AutoRecovery -|3 -|1 per zone - -|Function workers -|3 -|1 per zone - -|Admin -|1 -|1 per cluster - -|Heartbeat -|1 -|1 per cluster - -|=== - -== Next steps - -See xref:cluster-sizing-reference.adoc[] for specific deployment topologies and hardware recommendations. \ No newline at end of file diff --git a/modules/install-upgrade/pages/supported-versions.adoc b/modules/install-upgrade/pages/supported-versions.adoc deleted file mode 100644 index 55f6241..0000000 --- a/modules/install-upgrade/pages/supported-versions.adoc +++ /dev/null @@ -1,94 +0,0 @@ -= Supported software -:page-aliases: 2.10-1.x@install-upgrade:supported-versions.adoc, 2.10-2.x@install-upgrade:supported-versions.adoc - -Support covers only the following Software versions for {pulsar}: - -[NOTE] -==== -include::ROOT:partial$luna-name.adoc[] -==== - -[cols="2*"] -|=== -|Name |Versions - -|{pulsar} -|3.1.x -|=== - -Support covers only the following Software versions for {product} Distribution: - -[cols="2*"] -|=== -|Name |Version - -|{product} Distribution -|v.1.0.0 and above - -|=== - -Support covers only the following Software versions for the Open Source Projects: - -[cols="2*"] -|=== -|Name |Version - -|{pulsar-short} Admin Console -|v.1.0.0 and above - -|{pulsar-short} Heartbeat -|v.1.0.0 and above - -|{company} pulsar-sink -|v.1.0.0 and above - -|Starlight for JMS -|v.1.0.0 and above - -|{starlight-rabbitmq} -|v.1.0.0 and above - -|=== - -Support covers only the following {product} Connectors and {pulsar} Connectors (as included in the {pulsar} download): - -[cols="2*"] -|=== -|Name |Version - -|{cass-short} enhanced sink -|v1.6.11 -|Cloud storage sink -|v2.10.0 -|Elasticsearch sink -|v3.1.x -|HTTP sink -|v3.1.x -|JDBC-Clickhouse sink -|v3.1.x -|JDBC-MariaDB sink -|v3.1.x -|JDBC-Postgres sink -|v3.1.x -|JDBC-SQLite -|v3.1.x -|Kinesis sink -|v3.1.x -|Snowflake sink -|v0.1.13 -|{cass-short} source -|v2.2.9 -|Data generator source -|v3.1.x -|Debezium MongoDB source -|v3.1.x -|Debezium MySQL source -|v3.1.x -|Debezium Microsoft SQL Server Source -|v3.1.x -|Debezium Postgres source -|v3.1.x -|Kinesis source -|v3.1.x - -|=== \ No newline at end of file diff --git a/modules/install-upgrade/pages/upgrade-guide.adoc b/modules/install-upgrade/pages/upgrade-guide.adoc deleted file mode 100644 index ff2e034..0000000 --- a/modules/install-upgrade/pages/upgrade-guide.adoc +++ /dev/null @@ -1,454 +0,0 @@ -= Upgrade {product} from 2.10 to 3.1 -:navtitle: Upgrade from 2.10 to 3.1 - -This guide provides instructions and recommendations for upgrading {product} from version 2.10 to 3.1. - -[NOTE] -==== -include::ROOT:partial$luna-name.adoc[] -==== - -Upgrades to {product} 3.1 should only be performed from {product} 2.10. - -All {product} 2.10 components support the upgrade to 3.1. - -== Functional impacts - -This section describes changes in {product} 3.1 that may impact how your deployment functions. - -=== Default system topics - -In {pulsar-short} 3.1, system topics are now enabled by default. - -=== Prometheus metrics changes - -Prometheus metrics have been updated in {product} 3.1. - -* Prometheus Client version has changed from `0.5.0` to `0.16.0` -* Prometheus Metric type `UNTYPED` is renamed to `UNKNOWN` -* Metrics have been renamed because OpenMetrics's counter name needs a `_total` suffix - -.Renamed metrics -[cols="2,2"] -|=== -|{product} 2.10 |{product} 3.1 - -|pulsar_expired_token_count -|pulsar_expired_token_total - -|pulsar_authentication_success_count -|pulsar_authentication_success_total - -|pulsar_authentication_failures_count -|pulsar_authentication_failures_total - -|pulsar_source_received_total_1min -|pulsar_source_received_1min_total - -|pulsar_source_written_total_1min -|pulsar_source_written_1min_total - -|pulsar_source_source_total_1min -|pulsar_source_source_exceptions_1min_total - -|pulsar_source_system_exceptions_total_1min -|pulsar_source_system_exceptions_1min_total - -|pulsar_function_received_total_1min -|pulsar_function_received_1min_total - -|pulsar_function_user_exceptions_total_1min -|pulsar_function_user_exceptions_1min_total - -|pulsar_function_system_exceptions_total_1min -|pulsar_function_system_exceptions_1min_total - -|pulsar_function_processed_successfully_total_1min -|pulsar_function_processed_successfully_1min_total - -|pulsar_sink_received_total_1min -|pulsar_sink_received_1min_total - -|pulsar_sink_written_total_1min -|pulsar_sink_written_1min_total - -|pulsar_sink_sink_exceptions_total_1min -|pulsar_sink_sink_exceptions_1min_total - -|pulsar_sink_system_exceptions_total_1min -|pulsar_sink_system_exceptions_1min_total - -|pulsar_lb_unload_broker_count -|pulsar_lb_unload_broker_total - -|pulsar_lb_unload_bundle_count -|pulsar_lb_unload_bundle_total - -|pulsar_lb_bundles_split_count -|pulsar_lb_bundles_split_total - -|pulsar_schema_del_ops_failed_count -|pulsar_schema_del_ops_failed_total - -|pulsar_schema_get_ops_failed_count -|pulsar_schema_get_ops_failed_total - -|pulsar_schema_put_ops_failed_count -|pulsar_schema_put_ops_failed_total - -|pulsar_schema_compatible_count -|pulsar_schema_compatible_total - -|pulsar_schema_incompatible_count -|pulsar_schema_incompatible_total - -|pulsar_txn_committed_count -|pulsar_txn_committed_total - -|pulsar_txn_aborted_count -|pulsar_txn_aborted_total - -|pulsar_txn_created_count -|pulsar_txn_created_total - -|pulsar_txn_timeout_count -|pulsar_txn_timeout_total - -|pulsar_txn_append_log_count -|pulsar_txn_append_log_total -|=== - -The following PRs were merged to update metrics: - -* {apache-pulsar-repo}/pull/13785[#13785 - Bump prometheus client version from 0.5.0 to 0.15.0] -* {apache-pulsar-repo}/pull/16581[#16581 - Rename {pulsar-short} txn metrics to specify OpenMetrics] -* {apache-pulsar-repo}/pull/16610[#16610 - Rename {pulsar-short} schema metrics to specify OpenMetrics] -* {apache-pulsar-repo}/pull/16611[#16611 - Rename {pulsar-short} lb metrics to specify OpenMetrics] -* {apache-pulsar-repo}/pull/16591[#16591 - Bump prometheus client version from 0.15.0 to 0.16.0] -* {apache-pulsar-repo}/pull/17419[#17419 - Removed timestamp from all prometheus metrics.] - - -=== Other functional impacts - -The following PRs were merged in {product} 3.1 that may impact your deployment's functionality. - -[cols="1,2,3"] -|=== -|PR Link |Title |Functional Impact - -|{apache-pulsar-repo}/pull/19180[#19180] -|Deprecate blocking AuthorizationService, AuthorizationProvider methods -|This will affect the public API for the AuthorizationService and the AuthorizationProvider, which only impacts users that are running custom code inside the {pulsar-short} broker - -|{apache-pulsar-repo}/pull/19182[#19182] -|Remove AuthorizationProvider methods deprecated in 2.7 and 2.9 -|Removing deprecated methods allowTenantOperationAsync, allowTenantOperation, allowNamespaceOperationAsync, allowNamespaceOperation, allowNamespacePolicyOperationAsync, allowNamespacePolicyOperation, allowTopicOperationAsync, allowTopicOperation. These methods could be used by third party extensions - -|{apache-pulsar-repo}/pull/19197[#19197] -|Update AuthenticationProvider to simplify HTTP Authn -|This changes the public API within the broker as some methods are marked as @Deprecated - -|{apache-pulsar-repo}/pull/19295[#19295] -|OneStageAuth State: move authn out of constructor -|This could break 3rd party plugins in the broker if they were relying on authentication to happen in the constructor. In order to make those implementations fail fast, this PR includes a change to throw an exception when the getAuthRole is called without first calling authenticateAsync or authenticate. That makes these changes semi-backwards compatible. - -|{apache-pulsar-repo}/pull/19314[#19314] -|TokenAuthenticationState: authenticate token only once -|In a sense, this breaks an implicit contract that the class had. However, because the getAuthRole() method will throw an exception if called incorrectly, it is likely that misuse of this class will result in a fail fast behavior. - -|{apache-pulsar-repo}/pull/19455[#19455] -|Require authRole is proxyRole to set originalPrincipal -|This change affects the binary protocol's usage without changing the binary protocol itself. Upgrading existing proxies will not work if the proxyRoles is not correctly configured in the broker.conf. - -|{apache-pulsar-repo}/pull/19486[#19486] -|Remove default 30s ackTimeout when setting DLQ policy on java consumer -|Removed setting default ackTimeoutMillis in java ConsumerBuilder when a deadLetterPolicy is set. It has to be specified exclusively to use. -|=== - -== Configuration impacts - -This section describes changes in {product} 3.1 that may impact your deployment's configuration. - -=== Configuration values removed in 3.1 - -* {apache-pulsar-repo}/pull/14506[PR #14506] removes `managedLedgerNumWorkerThreads`. -The `MetadataStore` instance is now passed from the `PulsarService` directly to the `ManagedLedgerFactory`. - -* The {pulsar-short} SQL `conf/presto` directory has been removed. -** If you're upgrading {pulsar-short} SQL from 2.11 or earlier, copy the {pulsar-short} SQL config files from `conf/presto` to `trino/conf`. -** If you're downgrading {pulsar-short} SQL to 2.11 or earlier from newer versions, copy the {pulsar-short} SQL config files from `trino/conf` to `conf/presto`. - -=== Default values changed or deprecated in 3.1 - -The following default values in `broker.conf` and `standalone.conf` have changed or been deprecated in {product} 3.1. - -.`broker.conf` and `standalone.conf` values -[cols="1,1,1"] -|=== -|Configuration |{product} 2.10 Default | {product} 3.1 Default - -|`managedLedgerCacheEvictionFrequency`` -|`100.0` -|`0` - -|`managedLedgerMaxUnackedRangesToPersistInZooKeeper` -|`1000` -|`-1` - -|`systemTopicEnabled` -|`false` -|`true` - -|`topicLevelPoliciesEnabled` -|`false` -|`true` - -|`supportedNamespaceBundleSplitAlgorithms` -|`range_equally_divide`,`topic_count_equally_divide`,`specified_positions_divide` -|`range_equally_divide`,`topic_count_equally_divide`,`specified_positions_divide`,`flow_or_qps_equally_divide` - -|`loadBalancerDirectMemoryResourceWeight` -|`1.0` -|`0` - -|`fileSystemProfilePath` -|`../conf/filesystem_offload_core_site.xml` -|`conf/filesystem_offload_core_site.xml` - -|`gcsManagedLedgerOffloadMaxBlockSizeInBytes` -|`67108864` -|`134217728` -|=== - -== Operational impacts - -This section describes changes in {product} 3.1 that may impact how your deployment operates. - -=== JDK 17 upgrade - -{product} 3.1 uses JDK 17. - -The {pulsar-short} server module's `javac` release version is `17`. - -Client and client-server shared modules remain at the target Java 8 release. - -This modification is described in detail in {apache-pulsar-repo}/pull/15207[PIP-156]. - -=== Python 2 support removed - -{product} 3.1 removes Python 2 from build scripts. - -Python 3 is used in the build image. - -The build image is updated to use `ubuntu:20.04`, as there is no Python 3.7 support in the previous Ubuntu image. - -Executable scripts have been updated to invoke `python3` instead of `python`. - -This modification is described in detail in {apache-pulsar-repo}/pull/15376[PIP-155] - -== Known issues - -This section describes known issues encountered when upgrading to {product} 3.1. - -=== {bookkeeper-short} and RocksDB format - -[IMPORTANT] -==== -Downgrading to {product} 2.10 from {product} 3.1 is not supported for the {bookkeeper-reg} and {zookeeper-reg} components. -==== - -{pulsar-short} 3.1 uses RocksDB `7.x`, which writes in a format that is not compatible with RocksDB `6.x`. - -{product} 2.10 uses {bookkeeper} 4.14, which uses RocksDB `6.x`. - -All other components such as broker, proxy, and Functions Worker can be downgraded at any time. - -For more information, see {apache-pulsar-repo}/issues/22051[Issue 22051]. - -== Upgrade procedure - -{product} can be deployed on bare metal, Docker, and Kubernetes. - -This guide only addresses Kubernetes deployment. - -For more information on upgrading bare metal and Docker {pulsar-short} deployments, see the https://pulsar.apache.org/docs/3.3.x/administration-upgrade/[{pulsar-short} documentation]. - -=== Upgrade Kubernetes deployment with {kaap-short} - -Upgrade to {product} 3.1 on Kubernetes with {kaap}. - -For more information, see the xref:kaap-operator::index.adoc[{kaap-short} documentation]. - -. To prevent data loss, back up your existing {pulsar-short} data and configuration files. -. To save your current Helm release configuration, run the following command: -+ -[source,bash,subs="+quotes"] ----- -helm get values *RELEASE-NAME* > pulsar-backup-values.yaml ----- -+ -. To update the {product} Helm chart repository, run the following command: -+ -[source,bash] ----- -helm repo update ----- -+ -. Open `helm/kaap-stack/values.yaml`, and then update the image tag to `3.1.0` (or the specific tag you wish to use). -+ -[source,yaml] ----- -kaap: - enabled: true - cluster: - name: pulsar - create: true - spec: - global: - name: pulsar - image: - datastax/lunastreaming-all: 3.1_4.5 ----- -+ -. To modify other configurations, update `values.yaml` as needed. For example, to modify the broker's namespace shedding and splitting configurations, update the following fields: -+ -[source,yaml] ----- -kaap: - enabled: true - cluster: - name: pulsar - create: true - spec: - global: - name: pulsar - broker: - replicas: 2 - config: - loadBalancerNamespaceBundleSplitConditionHitCountThreshold: 1 - loadBalancerSheddingConditionHitCountThreshold: 1e ----- -+ -. To upgrade your existing {pulsar-short} installation, run the following Helm command. -This command assumes the default `pulsar` namespace. If you are using a different namespace, replace `pulsar` with your namespace. -The `--wait` flag ensures that Helm waits until all pods are ready before completing the upgrade. -+ -[source,bash,subs="+quotes"] ----- -helm upgrade --namespace *NAMESPACE* --wait --debug --timeout 1200s \ ---dependency-update pulsar *KAAP-REPO-DIRECTORY*/helm/kaap-stack \ ---values *PATH-TO-CURRENT-VALUES-FILE*.yaml ----- -+ -. Check the status of the pods to ensure they are running correctly: -+ -[source,bash,subs="+quotes"] ----- -kubectl get pods --namespace *NAMESPACE* ----- - -. Check the logs for any issues: -+ -[source,bash,subs="+quotes"] ----- -kubectl logs *POD-NAME* -n *NAMESPACE* ----- - -. After the upgrade, ensure all necessary configurations are in place and correct. - -=== Upgrade Kubernetes deployment with Helm chart - -The Helm chart for {product} is available in the {pulsar-helm-chart-repo}/blob/master/helm-chart-sources/pulsar/values.yaml[Helm chart sources] repository. - -. To prevent data loss, back up your existing {pulsar-short} data and configuration files. -. To save your current Helm release configuration, run the following command: -+ -[source,bash,subs="+quotes"] ----- -helm get values *RELEASE-NAME* > pulsar-backup-values.yaml ----- -+ -. To update the {product} Helm chart repository, run the following command: -+ -[source,bash] ----- -helm repo update ----- -+ -. Open `helm-chart-sources/pulsar/values.yaml` and update the image tag to `3.1.0` (or the specific tag you wish to use). -+ -[source,yaml] ----- -image: - broker: - # If not using tiered storage, you can use the smaller pulsar image - # for the broker - repository: datastax/lunastreaming-all - pullPolicy: IfNotPresent - tag: 3.1_4.5 - brokerSts: - # If not using tiered storage, you can use the smaller pulsar image - # for the broker - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latest - function: - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latest - zookeeper: - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latestupgr - bookkeeper: - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latest - proxy: - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latest - bastion: - repository: apachepulsar/pulsar - pullPolicy: IfNotPresent - tag: latest ----- -. Review and modify any other configuration parameters that may have changed between versions, such as resource limits, storage classes, and additional components. To modify other configurations, update `values.yaml` as needed. For example, to modify the broker's replica count, update the following fields: -+ -[source,yaml] ----- -broker: - component: broker - replicaCount: 2 - configData: - brokerDeduplicationEnabled: "false" ----- - -. To upgrade your existing {pulsar-short} installation, run the following Helm command. -This command assumes the default `pulsar` namespace. If you are using a different namespace, replace `pulsar` with your namespace. -The `--wait` flag ensures that Helm waits until all pods are ready before completing the upgrade. -+ -[source,bash,subs="+quotes"] ----- -helm upgrade --namespace *NAMESPACE* --wait --debug --timeout 1200s \ ---dependency-update pulsar *KAAP-REPO-DIRECTORY*/helm/kaap-stack \ ---values *PATH-TO-CURRENT-VALUES-FILE*.yaml ----- - -. To check the status of the pods to ensure they are running correctly, run the following command: -+ -[source,bash,subs="+quotes"] ----- -kubectl get pods --namespace *NAMESPACE* ----- - -. To check the logs for any issues, run the following command: -+ -[source,bash,subs="+quotes"] ----- -kubectl logs *POD-NAME* -n *NAMESPACE* ----- - -. After the upgrade, ensure all necessary configurations are in place and correct. - - From a35b80c94d4806349013a4cdc80f957180237f30 Mon Sep 17 00:00:00 2001 From: Mendon Kissling <59585235+mendonk@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:24:11 -0400 Subject: [PATCH 4/5] remove-kaap-references --- modules/ROOT/pages/faqs.adoc | 1 - modules/operations/pages/scale-cluster.adoc | 2 -- 2 files changed, 3 deletions(-) diff --git a/modules/ROOT/pages/faqs.adoc b/modules/ROOT/pages/faqs.adoc index e2406b4..54e7a57 100644 --- a/modules/ROOT/pages/faqs.adoc +++ b/modules/ROOT/pages/faqs.adoc @@ -56,7 +56,6 @@ The {product} distribution is designed for Java 17. However, because the product == What are the install options for {product}? -* **{kaap-short} Helm chart (Recommended)**: Use the {company} {kaap-operator-repo}[{kaap-short} Helm chart], which provides Kubernetes-native autoscaling and simplified management for {pulsar} clusters. For more information, see the xref:kaap-operator::index.adoc[{kaap}] documentation. * **{product} Helm chart**: Use the Helm chart provided at {pulsar-helm-chart-repo}[{pulsar-helm-chart-repo}] to install {product} in an existing Kubernetes cluster on your laptop or hosted by a cloud provider. * **Tarball**: Use the tarball provided at {pulsar-repo}/releases[{pulsar-repo}/releases] to install {product} on a server or VM. * **Ansible**: Use the {company} Ansible scripts provided at {pulsar-ansible-repo}[{pulsar-ansible-repo}] to install {product} on a server or VM with our provided playbooks. diff --git a/modules/operations/pages/scale-cluster.adoc b/modules/operations/pages/scale-cluster.adoc index 07653e5..564558a 100644 --- a/modules/operations/pages/scale-cluster.adoc +++ b/modules/operations/pages/scale-cluster.adoc @@ -7,8 +7,6 @@ This page will show you how to scale {product} clusters up for more compute capa include::ROOT:partial$luna-name.adoc[] ==== -include::operations:partial$operator-scaling.adoc[] - For our {pulsar-short} cluster installation, use this {pulsar-helm-chart-repo}[Helm chart]. To start the cluster, use the values provided in this {pulsar-helm-chart-repo}/blob/master/examples/dev-values.yaml[YAML file]. From ca447bf99753cb957cf65fdbd307fea1dfe7d7ff Mon Sep 17 00:00:00 2001 From: Mendon Kissling <59585235+mendonk@users.noreply.github.com> Date: Mon, 16 Mar 2026 13:42:08 -0400 Subject: [PATCH 5/5] fix-double-naming --- modules/install-upgrade/pages/quickstart-server-installs.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/install-upgrade/pages/quickstart-server-installs.adoc b/modules/install-upgrade/pages/quickstart-server-installs.adoc index 7161e18..859b831 100644 --- a/modules/install-upgrade/pages/quickstart-server-installs.adoc +++ b/modules/install-upgrade/pages/quickstart-server-installs.adoc @@ -22,7 +22,7 @@ The resulting {product} deployment includes: * JDK 11 + -{pulsar-short} can run with JDK8, but {company} {product} is designed for Java 11. +{pulsar-short} can run with JDK8, but {product} is designed for Java 11. * File System +