diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
deleted file mode 100644
index d11f8c5e518da..0000000000000
--- a/.github/workflows/ci-cpp.yaml
+++ /dev/null
@@ -1,103 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - CPP, Python Tests
-on:
- pull_request:
- branches:
- - branch-*
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- cpp-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: build package
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests
-
- - name: build cpp artifacts
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- echo "Build C++ client library"
- export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so"
- pulsar-client-cpp/docker-build.sh
-
- - name: run c++ tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: pulsar-client-cpp/docker-tests.sh
-
- - name: Upload test-logs
- uses: actions/upload-artifact@v2
- if: failure()
- continue-on-error: true
- with:
- name: test-logs
- path: test-logs
diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml
index a8b64dd182428..507936d838148 100644
--- a/bouncy-castle/bc/pom.xml
+++ b/bouncy-castle/bc/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
bouncy-castle-parent
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
bouncy-castle-bc
Apache Pulsar :: Bouncy Castle :: BC
-
${project.groupId}
@@ -39,23 +36,19 @@
${project.version}
provided
-
org.bouncycastle
bcpkix-jdk15on
${bouncycastle.version}
-
org.bouncycastle
bcprov-ext-jdk15on
${bouncycastle.version}
-
-
de.ntcomputer
@@ -73,7 +66,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
index 042572e352737..979d7fb083772 100644
--- a/bouncy-castle/bcfips-include-test/pom.xml
+++ b/bouncy-castle/bcfips-include-test/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
bouncy-castle-parent
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
bcfips-include-test
Pulsar Bouncy Castle FIPS Test
Broker and client runs auth include BC FIPS verison
-
${project.groupId}
@@ -39,7 +37,6 @@
${project.version}
test
-
${project.groupId}
pulsar-broker
@@ -53,7 +50,6 @@
test-jar
test
-
${project.groupId}
pulsar-broker
@@ -66,7 +62,6 @@
test
-
${project.groupId}
@@ -74,7 +69,6 @@
${project.version}
pkg
-
diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml
index dffbb6c04d132..a33ebddbfea2e 100644
--- a/bouncy-castle/bcfips/pom.xml
+++ b/bouncy-castle/bcfips/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
bouncy-castle-parent
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
bouncy-castle-bcfips
Apache Pulsar :: Bouncy Castle :: BC-FIPS
-
${project.groupId}
@@ -39,20 +36,17 @@
${project.version}
provided
-
org.bouncycastle
bc-fips
${bouncycastlefips.version}
-
org.bouncycastle
bcpkix-fips
${bouncycastlefips.version}
-
diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml
index f4b4c3c04e732..64bb5237626ca 100644
--- a/bouncy-castle/pom.xml
+++ b/bouncy-castle/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
pom
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
bouncy-castle-parent
Apache Pulsar :: Bouncy Castle :: Parent
-
bc
bcfips
-
bcfips-include-test
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index eac579be438f7..799f416c896c0 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
org.apache
apache
23
-
+
-
- org.apache.pulsar
+ io.streamnative
buildtools
- 2.10.5-SNAPSHOT
+ 2.10.5.3
jar
Pulsar Build Tools
-
1.8
1.8
@@ -50,10 +47,9 @@
32.0.0-jre
1.10.12
2.0
-
+
3.12.4
-
@@ -65,9 +61,7 @@
-
-
org.yaml
snakeyaml
@@ -88,7 +82,6 @@
guice
${guice.version}
-
org.testng
testng
@@ -145,7 +138,6 @@
${mockito.version}
-
@@ -240,6 +232,55 @@
**/proto/*
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.2.0
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ maven-gpg-plugin
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+
+
@@ -264,4 +305,14 @@
+
+
+ ossrh
+ https://s01.oss.sonatype.org/content/repositories/snapshots
+
+
+ ossrh
+ https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/
+
+
diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml
index 184f58487eaf0..9e12d754fe7b5 100644
--- a/buildtools/src/main/resources/log4j2.xml
+++ b/buildtools/src/main/resources/log4j2.xml
@@ -1,4 +1,4 @@
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml
index bb1e86e599292..b7acd43edb8bd 100644
--- a/buildtools/src/main/resources/pulsar/checkstyle.xml
+++ b/buildtools/src/main/resources/pulsar/checkstyle.xml
@@ -1,4 +1,4 @@
-
+
-
-
+
-
-
-
-
-
-
-
-
-
+
+
+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
+
+
+
+
+
-
-
-
-
+ IMPORT CHECKS
-
-
-
-
+ -->
+
+
+
+
-
-
-
-
-
+
+
+
+
+
+
+
-
-
-
-
+
+
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
} else
-->
-
else
-->
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
-
-
-
-
-
+
+
+
diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml
index 49cfa48d3c562..78fe2ca345964 100644
--- a/buildtools/src/main/resources/pulsar/suppressions.xml
+++ b/buildtools/src/main/resources/pulsar/suppressions.xml
@@ -1,4 +1,4 @@
-
+
-
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/conf/filesystem_offload_core_site.xml b/conf/filesystem_offload_core_site.xml
index d26cec2cc60f0..3c7e6d94bbe50 100644
--- a/conf/filesystem_offload_core_site.xml
+++ b/conf/filesystem_offload_core_site.xml
@@ -1,3 +1,4 @@
+
-
-
- fs.defaultFS
-
-
-
- hadoop.tmp.dir
- pulsar
-
-
- io.file.buffer.size
- 4096
-
-
- io.seqfile.compress.blocksize
- 1000000
-
-
- io.seqfile.compression.type
- BLOCK
-
-
- io.map.index.interval
- 128
-
-
+
+
+ fs.defaultFS
+
+
+
+ hadoop.tmp.dir
+ pulsar
+
+
+ io.file.buffer.size
+ 4096
+
+
+ io.seqfile.compress.blocksize
+ 1000000
+
+
+ io.seqfile.compression.type
+ BLOCK
+
+
+ io.map.index.interval
+ 128
+
diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml
index 190d9be92940b..84e6ce44f51e4 100644
--- a/conf/functions_log4j2.xml
+++ b/conf/functions_log4j2.xml
@@ -1,3 +1,4 @@
+
- pulsar-functions-instance
- 30
-
-
- pulsar.log.appender
- RollingFile
-
-
- pulsar.log.level
- info
-
-
- bk.log.level
- info
-
-
-
-
- Console
- SYSTEM_OUT
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- RollingFile
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz
- true
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- 1
- true
-
-
- 1 GB
-
-
- 0 0 0 * * ?
-
-
-
-
- ${sys:pulsar.function.log.dir}
- 2
-
- */${sys:pulsar.function.log.file}*log.gz
-
-
- 30d
-
-
-
-
-
- BkRollingFile
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz
- true
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- 1
- true
-
-
- 1 GB
-
-
- 0 0 0 * * ?
-
-
-
-
- ${sys:pulsar.function.log.dir}
- 2
-
- */${sys:pulsar.function.log.file}.bk*log.gz
-
-
- 30d
-
-
-
-
-
-
-
- org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
- ${sys:bk.log.level}
- false
-
- [BkRollingFile]
-
-
-
- info
-
- [${sys:pulsar.log.appender}]
- ${sys:pulsar.log.level}
-
-
-
-
\ No newline at end of file
+ pulsar-functions-instance
+ 30
+
+
+ pulsar.log.appender
+ RollingFile
+
+
+ pulsar.log.level
+ info
+
+
+ bk.log.level
+ info
+
+
+
+
+ Console
+ SYSTEM_OUT
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ RollingFile
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz
+ true
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ 1
+ true
+
+
+ 1 GB
+
+
+ 0 0 0 * * ?
+
+
+
+
+ ${sys:pulsar.function.log.dir}
+ 2
+
+ */${sys:pulsar.function.log.file}*log.gz
+
+
+ 30d
+
+
+
+
+
+ BkRollingFile
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz
+ true
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ 1
+ true
+
+
+ 1 GB
+
+
+ 0 0 0 * * ?
+
+
+
+
+ ${sys:pulsar.function.log.dir}
+ 2
+
+ */${sys:pulsar.function.log.file}.bk*log.gz
+
+
+ 30d
+
+
+
+
+
+
+
+ org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
+ ${sys:bk.log.level}
+ false
+
+ [BkRollingFile]
+
+
+
+ info
+
+ [${sys:pulsar.log.appender}]
+ ${sys:pulsar.log.level}
+
+
+
+
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 9c63d3eda7adc..1902247e551f4 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -335,3 +335,8 @@ zookeeperSessionTimeoutMs=-1
# ZooKeeper cache expiry time in seconds
# Deprecated: use metadataStoreCacheExpirySeconds
zooKeeperCacheExpirySeconds=-1
+
+### --- Metrics --- ###
+
+# Whether to enable the proxy's /metrics, /proxy-stats, and /status.html http endpoints
+enableProxyStatsEndpoints=true
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
index ee43b77099062..d2427afd32736 100644
--- a/distribution/io/pom.xml
+++ b/distribution/io/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
distribution
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-io-distribution
pom
Pulsar :: Distribution :: IO
-
${project.groupId}
@@ -46,7 +43,6 @@
provided
-
@@ -69,9 +65,32 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
-
generator-connector-config
@@ -151,5 +170,4 @@
-
diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml
index 1d8ea1261083f..690abc44b86e4 100644
--- a/distribution/io/src/assemble/io.xml
+++ b/distribution/io/src/assemble/io.xml
@@ -1,3 +1,4 @@
+
-
+
bin
dir
@@ -38,45 +37,111 @@
.
644
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
index 6a0b1deb75ca6..5253f655f82ed 100644
--- a/distribution/offloaders/pom.xml
+++ b/distribution/offloaders/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
distribution
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-offloader-distribution
pom
Pulsar :: Distribution :: Offloader
-
${project.groupId}
@@ -66,7 +63,6 @@
-
@@ -90,6 +86,30 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/distribution/offloaders/src/assemble/offloaders.xml b/distribution/offloaders/src/assemble/offloaders.xml
index 38f7eee906064..19c60f68c21d4 100644
--- a/distribution/offloaders/src/assemble/offloaders.xml
+++ b/distribution/offloaders/src/assemble/offloaders.xml
@@ -1,3 +1,4 @@
+
-
+
bin
tar.gz
@@ -38,13 +37,11 @@
.
644
-
offloaders
644
-
offloaders
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 5ce09b42d92e9..61e7e4bc73c40 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
distribution
pom
Pulsar :: Distribution
-
-
main
@@ -46,7 +42,6 @@
server
-
core-modules
@@ -54,7 +49,6 @@
-
@@ -64,6 +58,30 @@
${skipBuildDistribution}
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 4a7735cee4fb3..7b05f09fe11e3 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
distribution
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-server-distribution
pom
Pulsar :: Distribution :: Server
-
${project.groupId}
pulsar-broker
${project.version}
-
${project.groupId}
pulsar-proxy
${project.version}
-
${project.groupId}
pulsar-broker-auth-sasl
${project.version}
-
${project.groupId}
pulsar-client-auth-sasl
${project.version}
-
jline
jline
${jline.version}
-
org.apache.zookeeper
zookeeper-prometheus-metrics
${zookeeper.version}
-
${project.groupId}
pulsar-package-bookkeeper-storage
@@ -81,13 +72,11 @@
-
${project.groupId}
pulsar-package-filesystem-storage
${project.version}
-
${project.groupId}
pulsar-client-tools
@@ -99,75 +88,61 @@
-
${project.groupId}
pulsar-testclient
${project.version}
-
org.apache.logging.log4j
log4j-api
-
org.apache.logging.log4j
log4j-core
-
org.apache.logging.log4j
log4j-web
-
io.dropwizard.metrics
metrics-core
-
io.dropwizard.metrics
metrics-graphite
-
io.dropwizard.metrics
metrics-jvm
-
org.xerial.snappy
snappy-java
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
org.apache.logging.log4j
log4j-slf4j-impl
-
org.apache.bookkeeper.stats
prometheus-metrics-provider
-
io.prometheus
simpleclient_log4j2
-
${project.groupId}
bouncy-castle-bc
${project.version}
pkg
-
${project.groupId}
@@ -182,7 +157,6 @@
-
${project.groupId}
pulsar-functions-worker
@@ -199,7 +173,6 @@
-
${project.groupId}
@@ -214,7 +187,6 @@
-
${project.groupId}
@@ -227,19 +199,16 @@
-
io.grpc
grpc-all
-
io.perfmark
perfmark-api
compile
-
org.apache.bookkeeper.http
@@ -261,7 +230,6 @@
vertx-web
-
@@ -302,6 +270,30 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml
index 12c443dc6c564..c1675577f7ef8 100644
--- a/distribution/server/src/assemble/bin.xml
+++ b/distribution/server/src/assemble/bin.xml
@@ -1,3 +1,4 @@
+
-
+
bin
tar.gz
@@ -126,14 +125,11 @@
${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension}
-
- org.apache.pulsar:pulsar-functions-runtime-all
-
+ io.streamnative:pulsar-functions-runtime-all
org.projectlombok:lombok
-
- org.apache.pulsar:pulsar-functions-api-examples
+ io.streamnative:pulsar-functions-api-examples
*:tar.gz
diff --git a/docker/pom.xml b/docker/pom.xml
index 21f0a70d53116..97f2950ccced5 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
pom
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
docker-images
Apache Pulsar :: Docker Images
@@ -39,6 +38,30 @@
${skipBuildDistribution}
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 1a740f710f71c..e97af3177fa67 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -1,3 +1,4 @@
+
-
+
- org.apache.pulsar
+ io.streamnative
docker-images
- 2.10.5-SNAPSHOT
+ 2.10.5.3
4.0.0
pulsar-all-docker-image
Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components)
pom
-
${project.groupId}
@@ -64,7 +63,6 @@
-
docker
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 07967210438cd..4ceeeb21ef91c 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -1,3 +1,4 @@
+
-
+
- org.apache.pulsar
+ io.streamnative
docker-images
- 2.10.5-SNAPSHOT
+ 2.10.5.3
4.0.0
pulsar-docker-image
Apache Pulsar :: Docker Images :: Pulsar Latest Version
pom
-
false
false
-
${project.groupId}
@@ -51,7 +49,6 @@
-
docker
diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
index 30c5f64e7f858..81671e211cbba 100644
--- a/jclouds-shaded/pom.xml
+++ b/jclouds-shaded/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
jclouds-shaded
Apache Pulsar :: Jclouds shaded
-
-
org.apache.jclouds
jclouds-allblobstore
@@ -50,7 +45,6 @@
javax.annotation-api
-
@@ -66,7 +60,6 @@
true
true
false
-
com.google.guava:guava
@@ -88,7 +81,6 @@
com.google.errorprone:*
-
com.google
@@ -122,21 +114,44 @@
com.google.errorprone
org.apache.pulsar.jcloud.shade.com.google.errorprone
-
-
- org.apache.jclouds:jclouds-core
-
-
- lib/gson*jar
-
-
-
+
+ org.apache.jclouds:jclouds-core
+
+
+ lib/gson*jar
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml
index 2fa2a408031a6..90a4f25432577 100644
--- a/kafka-connect-avro-converter-shaded/pom.xml
+++ b/kafka-connect-avro-converter-shaded/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
- pulsar
- org.apache.pulsar
- 2.10.5-SNAPSHOT
- ..
-
-
- kafka-connect-avro-converter-shaded
- Apache Pulsar :: Kafka Connect Avro Converter shaded
-
-
-
-
- io.confluent
- kafka-connect-avro-converter
- ${confluent.version}
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
-
-
- ${shadePluginPhase}
-
- shade
-
-
-
-
- true
- true
-
-
-
- io.confluent:*
- io.confluent:kafka-avro-serializer
- io.confluent:kafka-schema-registry-client
- io.confluent:common-config
- io.confluent:common-utils
- org.apache.avro:*
-
- org.codehaus.jackson:jackson-core-asl
- org.codehaus.jackson:jackson-mapper-asl
- com.thoughtworks.paranamer:paranamer
- org.xerial.snappy:snappy-java
- org.apache.commons:commons-compress
- org.tukaani:xz
-
-
-
-
- io.confluent
- org.apache.pulsar.kafka.shade.io.confluent
-
-
- org.apache.avro
- org.apache.pulsar.kafka.shade.avro
-
-
- org.codehaus.jackson
- org.apache.pulsar.kafka.shade.org.codehaus.jackson
-
-
- com.thoughtworks.paranamer
- org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer
-
-
- org.xerial.snappy
- org.apache.pulsar.kafka.shade.org.xerial.snappy
-
-
- org.apache.commons
- org.apache.pulsar.kafka.shade.org.apache.commons
-
-
- org.tukaani
- org.apache.pulsar.kafka.shade.org.tukaani
-
-
-
-
-
-
-
-
-
-
-
+
+ 4.0.0
+
+ pulsar
+ io.streamnative
+ 2.10.5.3
+ ..
+
+ kafka-connect-avro-converter-shaded
+ Apache Pulsar :: Kafka Connect Avro Converter shaded
+
+
+
+ io.confluent
+ kafka-connect-avro-converter
+ ${confluent.version}
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ ${shadePluginPhase}
+
+ shade
+
+
+
+
+ true
+ true
+
+
+ io.confluent:*
+ io.confluent:kafka-avro-serializer
+ io.confluent:kafka-schema-registry-client
+ io.confluent:common-config
+ io.confluent:common-utils
+ org.apache.avro:*
+ org.codehaus.jackson:jackson-core-asl
+ org.codehaus.jackson:jackson-mapper-asl
+ com.thoughtworks.paranamer:paranamer
+ org.xerial.snappy:snappy-java
+ org.apache.commons:commons-compress
+ org.tukaani:xz
+
+
+
+
+ io.confluent
+ org.apache.pulsar.kafka.shade.io.confluent
+
+
+ org.apache.avro
+ org.apache.pulsar.kafka.shade.avro
+
+
+ org.codehaus.jackson
+ org.apache.pulsar.kafka.shade.org.codehaus.jackson
+
+
+ com.thoughtworks.paranamer
+ org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer
+
+
+ org.xerial.snappy
+ org.apache.pulsar.kafka.shade.org.xerial.snappy
+
+
+ org.apache.commons
+ org.apache.pulsar.kafka.shade.org.apache.commons
+
+
+ org.tukaani
+ org.apache.pulsar.kafka.shade.org.tukaani
+
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
+
+
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index 6d502d94abce9..330655f1cfd7b 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
managed-ledger
Managed Ledger
-
org.apache.bookkeeper
bookkeeper-server
-
org.apache.bookkeeper.stats
prometheus-metrics-provider
-
org.apache.bookkeeper.stats
codahale-metrics-provider
${bookkeeper.version}
-
com.google.protobuf
protobuf-java
-
${project.groupId}
pulsar-common
${project.version}
-
${project.groupId}
pulsar-metadata
${project.version}
-
com.google.guava
guava
-
${project.groupId}
testmocks
${project.version}
test
-
org.apache.zookeeper
zookeeper
@@ -85,29 +74,25 @@
test
- io.dropwizard.metrics
- metrics-core
- test
+ io.dropwizard.metrics
+ metrics-core
+ test
- org.xerial.snappy
- snappy-java
- test
+ org.xerial.snappy
+ snappy-java
+ test
-
org.awaitility
awaitility
test
-
org.slf4j
slf4j-api
-
-
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 173a8098d952d..b5b5cd21ff80b 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -139,6 +139,7 @@
import org.apache.pulsar.common.protocol.Commands;
import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
+import org.apache.pulsar.common.util.LazyLoadableValue;
import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap;
import org.apache.pulsar.metadata.api.Stat;
import org.slf4j.Logger;
@@ -2452,15 +2453,13 @@ private void maybeOffload(CompletableFuture finalPromise) {
}
}
- private boolean hasLedgerRetentionExpired(long ledgerTimestamp) {
- return config.getRetentionTimeMillis() >= 0
- && clock.millis() - ledgerTimestamp > config.getRetentionTimeMillis();
+ private boolean hasLedgerRetentionExpired(long retentionTimeMs, long ledgerTimestamp) {
+ return retentionTimeMs >= 0 && clock.millis() - ledgerTimestamp > retentionTimeMs;
}
- private boolean isLedgerRetentionOverSizeQuota(long sizeToDelete) {
+ private boolean isLedgerRetentionOverSizeQuota(long retentionSizeInMB, long totalSizeOfML, long sizeToDelete) {
// Handle the -1 size limit as "infinite" size quota
- return config.getRetentionSizeInMB() >= 0
- && TOTAL_SIZE_UPDATER.get(this) - sizeToDelete >= config.getRetentionSizeInMB() * MegaByte;
+ return retentionSizeInMB >= 0 && totalSizeOfML - sizeToDelete >= retentionSizeInMB * MegaByte;
}
boolean isOffloadedNeedsDelete(OffloadContext offload, Optional offloadPolicies) {
@@ -2518,6 +2517,11 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture> promise) {
}
long slowestReaderLedgerId = -1;
+ final LazyLoadableValue slowestNonDurationLedgerId =
+ new LazyLoadableValue(() -> getTheSlowestNonDurationReadPosition().getLedgerId());
+ final long retentionSizeInMB = config.getRetentionSizeInMB();
+ final long retentionTimeMs = config.getRetentionTimeMillis();
+ final long totalSizeOfML = TOTAL_SIZE_UPDATER.get(this);
if (!cursors.hasDurableCursors()) {
// At this point the lastLedger will be pointing to the
// ledger that has just been closed, therefore the +1 to
@@ -2540,7 +2544,10 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture> promise) {
long totalSizeToDelete = 0;
// skip ledger if retention constraint met
- for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) {
+ Iterator ledgerInfoIterator =
+ ledgers.headMap(slowestReaderLedgerId, false).values().iterator();
+ while (ledgerInfoIterator.hasNext()){
+ LedgerInfo ls = ledgerInfoIterator.next();
// currentLedger can not be deleted
if (ls.getLedgerId() == currentLedger.getId()) {
if (log.isDebugEnabled()) {
@@ -2560,8 +2567,9 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture> promise) {
}
totalSizeToDelete += ls.getSize();
- boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(totalSizeToDelete);
- boolean expired = hasLedgerRetentionExpired(ls.getTimestamp());
+ boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(retentionSizeInMB, totalSizeOfML,
+ totalSizeToDelete);
+ boolean expired = hasLedgerRetentionExpired(retentionTimeMs, ls.getTimestamp());
if (log.isDebugEnabled()) {
log.debug(
"[{}] Checking ledger {} -- time-old: {} sec -- "
@@ -2578,14 +2586,19 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture> promise) {
}
ledgersToDelete.add(ls);
} else {
- if (ls.getLedgerId() < getTheSlowestNonDurationReadPosition().getLedgerId()) {
- // once retention constraint has been met, skip check
- if (log.isDebugEnabled()) {
- log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name,
- ls.getLedgerId());
- }
- invalidateReadHandle(ls.getLedgerId());
+ // once retention constraint has been met, skip check
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, ls.getLedgerId());
}
+ releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue());
+ break;
+ }
+ }
+
+ while (ledgerInfoIterator.hasNext()) {
+ LedgerInfo ls = ledgerInfoIterator.next();
+ if (!releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue())) {
+ break;
}
}
@@ -2686,6 +2699,21 @@ public void operationFailed(MetaStoreException e) {
}
}
+ /**
+ * @param ledgerId the ledger handle which maybe will be released.
+ * @return if the ledger handle was released.
+ */
+ private boolean releaseReadHandleIfNoLongerRead(long ledgerId, long slowestNonDurationLedgerId) {
+ if (ledgerId < slowestNonDurationLedgerId) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Ledger {} no longer needs to be read, close the cached readHandle", name, ledgerId);
+ }
+ invalidateReadHandle(ledgerId);
+ return true;
+ }
+ return false;
+ }
+
/**
* Non-durable cursors have to be moved forward when data is trimmed since they are not retain that data.
* This method also addresses a corner case for durable cursors in which the cursor is caught up, i.e. the mark
diff --git a/pom.xml b/pom.xml
index 90297c75c2985..9f71cab2f2258 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
pom
@@ -28,30 +27,24 @@
apache
23
-
- org.apache.pulsar
+ io.streamnative
pulsar
-
- 2.10.5-SNAPSHOT
-
+ 2.10.5.3
Pulsar
Pulsar is a distributed pub-sub messaging platform with a very
flexible messaging model and an intuitive client API.
https://github.com/apache/pulsar
-
Apache Software Foundation
http://www.apache.org/
2017
-
Apache Pulsar developers
http://pulsar.apache.org/
-
Apache License, Version 2.0
@@ -59,56 +52,50 @@ flexible messaging model and an intuitive client API.
repo
-
- https://github.com/apache/pulsar
- scm:git:https://github.com/apache/pulsar.git
- scm:git:ssh://git@github.com:apache/pulsar.git
+ https://github.com/streamnative/pulsar
+ scm:git:https://github.com/streamnative/pulsar.git
+ scm:git:ssh://git@github.com:streamnative/pulsar.git
-
GitHub Actions
https://github.com/apache/pulsar/actions
-
Github
https://github.com/apache/pulsar/issues
-
8
8
-
**/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java
quarantine
-
UTF-8
UTF-8
true
-
+
true
4
false
1
-
+
apachepulsar
false
false
package
package
-
1.21
-
4.14.7
3.6.4
1.5.0
- 1.1.10.1
- 3.2.5
+ 1.1.10.1
+
+ 3.2.5
+
5.1.0
4.1.93.Final
9.4.51.v20230217
@@ -212,13 +199,11 @@ flexible messaging model and an intuitive client API.
1.10.12
5.3.3
2.0.6
-
3.6.0
3.4.0
1.15.3
2.2
-
3.2.8
1.1.1
@@ -234,7 +219,6 @@ flexible messaging model and an intuitive client API.
1.5.4
5.4.0
2.33.2
-
0.6.1
3.0.0
@@ -267,14 +251,11 @@ flexible messaging model and an intuitive client API.
0.4
8.0.1
0.9.44
-
rename-netty-native-libs.sh
-
-
org.asynchttpclient
async-http-client
@@ -290,57 +271,49 @@ flexible messaging model and an intuitive client API.
-
org.testng
testng
${testng.version}
-
- org.yaml
- *
-
+
+ org.yaml
+ *
+
-
org.hamcrest
hamcrest
${hamcrest.version}
test
-
junit
junit
${junit4.version}
-
org.awaitility
awaitility
${awaitility.version}
test
-
org.mockito
mockito-core
${mockito.version}
-
org.mockito
mockito-inline
${mockito.version}
-
org.powermock
powermock-reflect
${powermock.version}
-
org.apache.zookeeper
zookeeper
@@ -402,7 +375,6 @@ flexible messaging model and an intuitive client API.
-
org.apache.bookkeeper
bookkeeper-server
@@ -431,13 +403,11 @@ flexible messaging model and an intuitive client API.
-
org.apache.bookkeeper
cpu-affinity
${bookkeeper.version}
-
io.vertx
vertx-core
@@ -448,38 +418,33 @@ flexible messaging model and an intuitive client API.
vertx-web
${vertx.version}
-
- org.apache.curator
- curator-recipes
- ${curator.version}
-
-
- org.apache.zookeeper
- *
-
-
+ org.apache.curator
+ curator-recipes
+ ${curator.version}
+
+
+ org.apache.zookeeper
+ *
+
+
-
org.apache.bookkeeper
bookkeeper-common-allocator
${bookkeeper.version}
-
org.apache.bookkeeper
bookkeeper-tools-framework
${bookkeeper.version}
-
org.reflections
reflections
${reflections.version}
-
org.apache.bookkeeper
@@ -508,7 +473,6 @@ flexible messaging model and an intuitive client API.
-
org.apache.bookkeeper
@@ -541,19 +505,16 @@ flexible messaging model and an intuitive client API.
-
org.apache.bookkeeper
bookkeeper-common
${bookkeeper.version}
-
org.apache.bookkeeper.stats
bookkeeper-stats-api
${bookkeeper.version}
-
org.apache.bookkeeper.stats
datasketches-metrics-provider
@@ -565,37 +526,31 @@ flexible messaging model and an intuitive client API.
-
org.apache.bookkeeper.stats
prometheus-metrics-provider
${bookkeeper.version}
-
org.rocksdb
rocksdbjni
${rocksdb.version}
-
org.eclipse.jetty
jetty-server
${jetty.version}
-
org.eclipse.jetty
jetty-alpn-conscrypt-server
${jetty.version}
-
org.conscrypt
conscrypt-openjdk-uber
${conscrypt.version}
-
org.eclipse.jetty
jetty-bom
@@ -603,7 +558,6 @@ flexible messaging model and an intuitive client API.
pom
import
-
io.netty
netty-bom
@@ -611,79 +565,66 @@ flexible messaging model and an intuitive client API.
pom
import
-
com.beust
jcommander
${jcommander.version}
-
com.google.guava
guava
${guava.version}
-
com.google.inject
guice
${guice.version}
-
com.google.inject.extensions
guice-assistedinject
${guice.version}
-
org.apache.commons
commons-lang3
${commons-lang3.version}
-
org.apache.commons
commons-compress
${commons-compress.version}
-
commons-configuration
commons-configuration
${commons-configuration.version}
-
commons-io
commons-io
${commons-io.version}
-
org.slf4j
slf4j-api
${slf4j.version}
-
org.slf4j
slf4j-simple
${slf4j.version}
-
org.slf4j
jul-to-slf4j
${slf4j.version}
-
org.slf4j
jcl-over-slf4j
${slf4j.version}
-
org.apache.logging.log4j
log4j-bom
@@ -691,49 +632,41 @@ flexible messaging model and an intuitive client API.
pom
import
-
commons-codec
commons-codec
${commons-codec.version}
-
org.glassfish.jersey.core
jersey-server
${jersey.version}
-
org.glassfish.jersey.core
jersey-client
${jersey.version}
-
org.glassfish.jersey.inject
jersey-hk2
${jersey.version}
-
org.glassfish.jersey.containers
jersey-container-servlet-core
${jersey.version}
-
org.glassfish.jersey.containers
jersey-container-servlet
${jersey.version}
-
javax.ws.rs
javax.ws.rs-api
${javax.ws.rs-api.version}
-
org.glassfish.jersey.media
jersey-media-json-jackson
@@ -745,23 +678,20 @@ flexible messaging model and an intuitive client API.
-
org.glassfish.jersey.media
jersey-media-multipart
${jersey.version}
-
net.java.dev.jna
jna
${jna.version}
-
- com.github.docker-java
- docker-java-core
- ${docker-java.version}
+ com.github.docker-java
+ docker-java-core
+ ${docker-java.version}
com.github.docker-java
@@ -773,7 +703,6 @@ flexible messaging model and an intuitive client API.
docker-java-transport-zerodep
${docker-java.version}
-
com.fasterxml.jackson
jackson-bom
@@ -781,122 +710,101 @@ flexible messaging model and an intuitive client API.
pom
import
-
org.codehaus.jettison
jettison
${jettison.version}
-
com.fasterxml.woodstox
woodstox-core
${woodstox.version}
-
-
org.hdrhistogram
HdrHistogram
${hdrHistogram.version}
-
io.swagger
swagger-core
${swagger.version}
-
io.swagger
swagger-annotations
${swagger.version}
-
javax.servlet
javax.servlet-api
${javax.servlet-api}
-
com.github.ben-manes.caffeine
caffeine
${caffeine.version}
-
com.yahoo.athenz
athenz-zts-java-client-core
${athenz.version}
-
com.yahoo.athenz
athenz-zpe-java-client
${athenz.version}
-
com.github.zafarkhaja
java-semver
${java-semver.version}
-
io.prometheus
simpleclient
${prometheus.version}
-
io.prometheus
simpleclient_hotspot
${prometheus.version}
-
io.prometheus
simpleclient_log4j2
${prometheus.version}
-
io.prometheus
simpleclient_servlet
${prometheus.version}
-
io.prometheus
simpleclient_jetty
${prometheus.version}
-
io.prometheus
simpleclient_caffeine
${prometheus.version}
-
com.carrotsearch
hppc
${hppc.version}
-
io.etcd
jetcd-core
${jetcd.version}
-
io.etcd
jetcd-test
${jetcd.version}
-
org.apache.spark
spark-streaming_2.10
@@ -924,7 +832,6 @@ flexible messaging model and an intuitive client API.
-
io.jsonwebtoken
jjwt-api
@@ -940,13 +847,11 @@ flexible messaging model and an intuitive client API.
jjwt-jackson
${jsonwebtoken.version}
-
net.jodah
typetools
${typetools.version}
-
io.grpc
grpc-bom
@@ -954,7 +859,6 @@ flexible messaging model and an intuitive client API.
pom
import
-
io.grpc
grpc-all
@@ -978,25 +882,21 @@ flexible messaging model and an intuitive client API.
-
com.google.http-client
google-http-client
${google-http-client.version}
-
com.google.http-client
google-http-client-jackson2
${google-http-client.version}
-
com.google.http-client
google-http-client-gson
${google-http-client.version}
-
io.perfmark
perfmark-api
@@ -1009,7 +909,6 @@ flexible messaging model and an intuitive client API.
-
com.google.protobuf
protobuf-bom
@@ -1017,19 +916,16 @@ flexible messaging model and an intuitive client API.
pom
import
-
com.google.code.gson
gson
${gson.version}
-
com.yahoo.datasketches
sketches-core
${sketches.version}
-
com.amazonaws
aws-java-sdk-bom
@@ -1037,7 +933,6 @@ flexible messaging model and an intuitive client API.
pom
import
-
org.apache.distributedlog
distributedlog-core
@@ -1050,13 +945,11 @@ flexible messaging model and an intuitive client API.
-
commons-collections
commons-collections
${commons.collections.version}
-
com.lmax
@@ -1080,97 +973,81 @@ flexible messaging model and an intuitive client API.
assertj-core
${assertj-core.version}
-
org.projectlombok
lombok
${lombok.version}
-
javax.annotation
javax.annotation-api
${javax.annotation-api.version}
-
javax.xml.bind
jaxb-api
${jaxb-api}
-
jakarta.xml.bind
jakarta.xml.bind-api
${jakarta.xml.bind.version}
-
com.sun.activation
javax.activation
${javax.activation.version}
-
com.sun.activation
jakarta.activation
${jakarta.activation.version}
-
jakarta.activation
jakarta.activation-api
${jakarta.activation.version}
-
jakarta.validation
jakarta.validation-api
${jakarta.validation.version}
-
io.opencensus
opencensus-api
${opencensus.version}
-
io.opencensus
opencensus-contrib-http-util
${opencensus.version}
-
io.opencensus
opencensus-contrib-grpc-metrics
${opencensus.version}
-
org.opensearch.client
opensearch-rest-high-level-client
${opensearch.version}
-
joda-time
joda-time
${joda.version}
-
org.javassist
javassist
${javassist.version}
-
net.jcip
jcip-annotations
${jcip.version}
-
io.airlift
aircompressor
@@ -1182,19 +1059,16 @@ flexible messaging model and an intuitive client API.
-
org.objenesis
objenesis
${objenesis.version}
-
org.apache.httpcomponents
httpclient
${apache-http-client.version}
-
com.github.spotbugs
spotbugs-annotations
@@ -1202,31 +1076,26 @@ flexible messaging model and an intuitive client API.
provided
true
-
com.google.errorprone
error_prone_annotations
${errorprone.version}
-
com.google.j2objc
j2objc-annotations
${j2objc-annotations.version}
-
org.yaml
snakeyaml
${snakeyaml.version}
-
org.apache.ant
ant
${ant.version}
-
com.squareup.okhttp3
okhttp
@@ -1247,7 +1116,6 @@ flexible messaging model and an intuitive client API.
okio
${okio.version}
-
org.jetbrains.kotlin
kotlin-stdlib
@@ -1258,92 +1126,77 @@ flexible messaging model and an intuitive client API.
kotlin-stdlib-common
${kotlin-stdlib.version}
-
org.jetbrains.kotlin
kotlin-stdlib-jdk8
${kotlin-stdlib.version}
-
- com.typesafe.netty
- netty-reactive-streams
- ${netty-reactive-streams.version}
+ com.typesafe.netty
+ netty-reactive-streams
+ ${netty-reactive-streams.version}
-
ch.qos.reload4j
reload4j
${reload4j.version}
-
org.roaringbitmap
RoaringBitmap
${roaringbitmap.version}
-
org.scala-lang
scala-library
${scala-library.version}
-
org.scala-lang
scala-reflect
${scala-library.version}
-
-
- org.apache.pulsar
+ io.streamnative
buildtools
${project.version}
test
-
org.testng
testng
test
-
org.mockito
mockito-core
test
-
org.mockito
mockito-inline
test
-
com.github.stefanbirkner
system-lambda
${system-lambda.version}
test
-
org.powermock
powermock-reflect
test
-
org.assertj
assertj-core
test
-
org.projectlombok
lombok
@@ -1354,7 +1207,6 @@ flexible messaging model and an intuitive client API.
javax.annotation-api
provided
-
org.apache.bookkeeper
@@ -1364,8 +1216,8 @@ flexible messaging model and an intuitive client API.
tests
- org.bouncycastle
- *
+ org.bouncycastle
+ *
org.slf4j
@@ -1385,8 +1237,12 @@ flexible messaging model and an intuitive client API.
+
+ org.apache.logging.log4j
+ log4j-layout-template-json
+ 2.20.0
+
-
${project.artifactId}
@@ -1395,7 +1251,7 @@ flexible messaging model and an intuitive client API.
maven-checkstyle-plugin
- org.apache.pulsar
+ io.streamnative
buildtools
${project.version}
@@ -1458,7 +1314,6 @@ flexible messaging model and an intuitive client API.
-
org.commonjava.maven.plugins
directory-maven-plugin
@@ -1473,14 +1328,13 @@ flexible messaging model and an intuitive client API.
pulsar.basedir
- org.apache.pulsar
+ io.streamnative
pulsar
-
com.mycila
license-maven-plugin
@@ -1591,36 +1445,28 @@ flexible messaging model and an intuitive client API.
src/assemble/README.bin.txt
src/assemble/LICENSE.bin.txt
src/assemble/NOTICE.bin.txt
-
src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java
src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java
bin/proto/MLDataFormats_pb2.py
-
**/avro/generated/*.java
-
**/*.avsc
-
src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java
src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java
src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionKey.java
src/main/java/org/apache/pulsar/io/kinesis/fbs/KeyValue.java
src/main/java/org/apache/pulsar/io/kinesis/fbs/Message.java
-
src/main/java/org/apache/bookkeeper/mledger/util/AbstractCASReferenceCounted.java
-
dependency-reduced-pom.xml
-
pulsar-client-cpp/lib/lz4/lz4.*
-
pulsar-client-go/go.mod
pulsar-client-go/go.sum
@@ -1628,20 +1474,16 @@ flexible messaging model and an intuitive client API.
pulsar-function-go/go.sum
pulsar-function-go/examples/go.mod
pulsar-function-go/examples/go.sum
-
pulsar-client-cpp/lib/checksum/crc32c_sw.cc
-
pulsar-client-cpp/generated/**
**/META-INF/services/com.scurrilous.circe.HashProvider
**/META-INF/services/io.prestosql.spi.Plugin
-
**/django/stats/migrations/*.py
**/conf/uwsgi_params
-
**/*.crt
**/*.key
@@ -1654,21 +1496,16 @@ flexible messaging model and an intuitive client API.
certificate-authority/index.txt
certificate-authority/serial
certificate-authority/README.md
-
**/zk-3.5-test-data/*
-
**/requirements.txt
-
conf/schema_example.conf
**/templates/*.tpl
-
**/.helmignore
**/_helpers.tpl
-
*.md
.github/**
@@ -1714,6 +1551,7 @@ flexible messaging model and an intuitive client API.
**/*.so.*
**/*.dylib
src/test/resources/*.txt
+ **/dependency-reduced-pom.xml
@@ -1722,54 +1560,106 @@ flexible messaging model and an intuitive client API.
maven-enforcer-plugin
${maven-enforcer-plugin.version}
-
- enforce-maven
-
- enforce
-
-
-
-
- [1.8.0,)
-
-
- [3.3.9,)
-
-
-
-
+
+ enforce-maven
+
+ enforce
+
+
+
+
+ [1.8.0,)
+
+
+ [3.3.9,)
+
+
+
+
- org.apache.maven.plugins
- maven-assembly-plugin
- ${maven-assembly-plugin.version}
- false
-
-
- source-release-assembly-tar-gz
- generate-sources
-
- single
-
-
- ${skipSourceReleaseAssembly}
- true
-
- src/assembly-source-package.xml
-
- apache-pulsar-${project.version}-src
- false
-
- tar.gz
-
- posix
-
-
-
+ org.apache.maven.plugins
+ maven-assembly-plugin
+ ${maven-assembly-plugin.version}
+ false
+
+
+ source-release-assembly-tar-gz
+ generate-sources
+
+ single
+
+
+ ${skipSourceReleaseAssembly}
+ true
+
+ src/assembly-source-package.xml
+
+ apache-pulsar-${project.version}-src
+ false
+
+ tar.gz
+
+ posix
+
+
+
+
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.13
+ true
+
+ ossrh
+ https://s01.oss.sonatype.org/
+ 60285aee9d4161
+ true
+ false
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.2.0
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+
+ attach-javadocs
+
+ jar
+
+
+ none
+ false
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+ 3.0.0-M1
+
+ true
+ false
+ release
+ deploy
+
-
@@ -1787,10 +1677,10 @@ flexible messaging model and an intuitive client API.
maven-surefire-plugin
- ${include}
+ ${include}
- **/*$*,${exclude}
+ **/*$*,${exclude}
${groups}
${excludedGroups}
@@ -1874,19 +1764,29 @@ flexible messaging model and an intuitive client API.
com.github.spotbugs
spotbugs-maven-plugin
${spotbugs-maven-plugin.version}
-
-
- com.github.spotbugs
- spotbugs
- ${spotbugs.version}
-
-
+
+
+ com.github.spotbugs
+ spotbugs
+ ${spotbugs.version}
+
+
org.codehaus.mojo
properties-maven-plugin
${properties-maven-plugin.version}
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.13
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 3.2.0
+
@@ -1902,7 +1802,6 @@ flexible messaging model and an intuitive client API.
-
jdk11
@@ -2012,7 +1911,6 @@ flexible messaging model and an intuitive client API.
tests
-
contrib-check
-
-
-
- org.apache.rat
- apache-rat-plugin
-
-
-
- check
-
- verify
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- check-style
- verify
-
- ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml
- ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml
- UTF-8
- **/proto/*
-
-
- check
-
-
-
-
-
-
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+
+
+
+ check
+
+ verify
+
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ check-style
+ verify
+
+ ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml
+ ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml
+ UTF-8
+ **/proto/*
+
+
+ check
+
+
+
+
+
+
-
windows
@@ -2068,7 +1965,6 @@ flexible messaging model and an intuitive client API.
rename-netty-native-libs.cmd
-
@@ -2110,40 +2006,29 @@ flexible messaging model and an intuitive client API.
pulsar-broker-auth-sasl
pulsar-client-auth-sasl
pulsar-config-validation
-
structured-event-log
-
pulsar-transaction
-
pulsar-functions
-
pulsar-io
-
kafka-connect-avro-converter-shaded
-
bouncy-castle
-
pulsar-client-messagecrypto-bc
-
pulsar-metadata
jclouds-shaded
-
pulsar-package-management
-
distribution
docker
tests
-
core-modules
@@ -2169,31 +2054,23 @@ flexible messaging model and an intuitive client API.
pulsar-broker-auth-sasl
pulsar-client-auth-sasl
pulsar-config-validation
-
pulsar-transaction
-
pulsar-functions
-
pulsar-io
-
bouncy-castle
-
pulsar-client-messagecrypto-bc
-
distribution
pulsar-metadata
-
pulsar-package-management
-
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-broker-auth-athenz
jar
Athenz authentication plugin for broker
-
-
${project.groupId}
pulsar-broker
${project.version}
-
${project.groupId}
testmocks
${project.version}
test
-
com.yahoo.athenz
athenz-zpe-java-client
-
-
@@ -74,7 +65,6 @@
-
com.github.spotbugs
spotbugs-maven-plugin
@@ -92,7 +82,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
@@ -106,7 +95,6 @@
-
diff --git a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml
+++ b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml
index 5a4186bd694ad..54833f9bd9ce2 100644
--- a/pulsar-broker-auth-sasl/pom.xml
+++ b/pulsar-broker-auth-sasl/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-broker-auth-sasl
jar
SASL authentication plugin for broker
-
-
${project.groupId}
pulsar-broker
${project.version}
-
org.apache.kerby
kerby-config
@@ -53,7 +47,6 @@
-
org.apache.kerby
kerb-simplekdc
@@ -66,30 +59,25 @@
-
${project.groupId}
pulsar-proxy
${project.version}
test
-
${project.groupId}
testmocks
${project.version}
test
-
${project.groupId}
pulsar-client-auth-sasl
${project.version}
test
-
-
@@ -111,8 +99,6 @@
-
-
@@ -132,7 +118,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java
index 6ded38132e1ad..9d90cc26ec0a0 100644
--- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java
+++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java
@@ -76,7 +76,7 @@ public String verifyAndExtract(String signedStr) throws AuthenticationException
String originalSignature = signedStr.substring(index + SIGNATURE.length());
String rawValue = signedStr.substring(0, index);
String currentSignature = computeSignature(rawValue);
- if (!originalSignature.equals(currentSignature)) {
+ if (!MessageDigest.isEqual(originalSignature.getBytes(), currentSignature.getBytes())){
throw new AuthenticationException("Invalid signature");
}
return rawValue;
diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml
index ca7407c5ec51e..cfef958671cf9 100644
--- a/pulsar-broker-common/pom.xml
+++ b/pulsar-broker-common/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-broker-common
Common classes used in multiple broker modules
-
${project.groupId}
pulsar-metadata
${project.version}
-
com.google.guava
guava
-
io.prometheus
simpleclient_jetty
-
javax.servlet
javax.servlet-api
-
javax.ws.rs
javax.ws.rs-api
-
io.jsonwebtoken
jjwt-impl
-
io.jsonwebtoken
jjwt-jackson
-
org.bouncycastle
@@ -76,14 +65,12 @@
${bouncycastlefips.version}
test
-
org.awaitility
awaitility
test
-
@@ -103,7 +90,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java
index bc670c53a8c8f..6f3fb7d714a70 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java
@@ -92,6 +92,13 @@ protected CompletableFuture> getAsync(String path) {
return cache.get(path);
}
+ protected CompletableFuture> refreshAndGetAsync(String path) {
+ return store.sync(path).thenCompose(___ -> {
+ cache.invalidate(path);
+ return cache.get(path);
+ });
+ }
+
protected void set(String path, Function modifyFunction) throws MetadataStoreException {
try {
setAsync(path, modifyFunction).get(operationTimeoutSec, TimeUnit.SECONDS);
diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
index 74d4bcf6c472e..5e59d20bfbfbb 100644
--- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
+++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java
@@ -122,6 +122,13 @@ public Optional getPolicies(NamespaceName ns) throws MetadataStoreExce
return get(joinPath(BASE_POLICIES_PATH, ns.toString()));
}
+ /**
+ * Get the namespace policy from the metadata cache. This method will not trigger the load of metadata cache.
+ *
+ * @deprecated Since this method may introduce inconsistent namespace policies. we should use
+ * #{@link NamespaceResources#getPoliciesAsync}
+ */
+ @Deprecated
public Optional getPoliciesIfCached(NamespaceName ns) {
return getCache().getIfCached(joinPath(BASE_POLICIES_PATH, ns.toString()));
}
@@ -230,8 +237,18 @@ public CompletableFuture> listPartitionedTopicsAsync(NamespaceName
}
public CompletableFuture> getPartitionedTopicMetadataAsync(TopicName tn) {
- return getAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(),
- tn.getEncodedLocalName()));
+ return getPartitionedTopicMetadataAsync(tn, false);
+ }
+
+ public CompletableFuture> getPartitionedTopicMetadataAsync(TopicName tn,
+ boolean refresh) {
+ if (refresh) {
+ return refreshAndGetAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(),
+ tn.getEncodedLocalName()));
+ } else {
+ return getAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(),
+ tn.getEncodedLocalName()));
+ }
}
public boolean partitionedTopicExists(TopicName tn) throws MetadataStoreException {
diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml
index 3e5179c8b69c1..91fd0af562af3 100644
--- a/pulsar-broker-shaded/pom.xml
+++ b/pulsar-broker-shaded/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-broker-shaded
jar
Pulsar Shaded Broker
-
${project.groupId}
@@ -41,7 +37,6 @@
${project.parent.version}
-
@@ -70,7 +65,6 @@
-
maven-antrun-plugin
@@ -83,15 +77,12 @@
-
+
-
org.apache.maven.plugins
maven-shade-plugin
@@ -105,10 +96,9 @@
true
true
false
-
- org.apache.pulsar:*
+ io.streamnative:*
org.apache.bookkeeper*:*
commons-*:*
org.apache.commons:*
@@ -119,10 +109,9 @@
com.google.*:*
com.fasterxml.jackson.*:*
io.netty:*
- org.apache.pulsar:pulsar-common
+ io.streamnative:pulsar-common
org.apache.bookkeeper:circe-checksum
com.yahoo.datasketches:sketches-core
-
javax.ws.rs:*
javax.websocket:*
org.glassfish.hk2*:*
@@ -138,7 +127,6 @@
org.apache.zookeeper:*
jline:*
javax.servlet:*
-
com.beust:*
io.swagger:*
joda-time:*
@@ -154,7 +142,7 @@
- org.apache.pulsar:pulsar-client-original
+ io.streamnative:pulsar-client-original
**
@@ -205,7 +193,6 @@
com.yahoo.sketches
org.apache.pulsar.shade.com.yahoo.sketches
-
org.apache.zookeeper
org.apache.pulsar.shade.org.apache.zookeeper
@@ -372,6 +359,30 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
index 4bd6bda8a2af3..d22d56a5e5aed 100644
--- a/pulsar-broker/pom.xml
+++ b/pulsar-broker/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-broker
jar
Pulsar Broker
-
commons-codec
commons-codec
-
commons-collections
commons-collections
-
org.apache.commons
commons-lang3
-
org.slf4j
slf4j-api
-
io.netty
netty-transport
-
com.google.protobuf
protobuf-java
-
${project.groupId}
pulsar-client-original
${project.version}
-
${project.groupId}
pulsar-websocket
${project.version}
-
${project.groupId}
pulsar-client-admin-original
${project.version}
-
${project.groupId}
managed-ledger
${project.version}
-
org.apache.curator
curator-recipes
-
org.apache.bookkeeper
stream-storage-server
@@ -123,242 +109,197 @@
-
org.apache.bookkeeper
bookkeeper-tools-framework
-
${project.groupId}
pulsar-broker-common
${project.version}
-
${project.groupId}
pulsar-transaction-common
${project.version}
-
${project.groupId}
pulsar-io-batch-discovery-triggerers
${project.version}
test
-
${project.groupId}
testmocks
${project.version}
test
-
com.github.tomakehurst
wiremock-jre8
${wiremock.version}
test
-
- io.dropwizard.metrics
- metrics-core
+ io.dropwizard.metrics
+ metrics-core
-
- org.xerial.snappy
- snappy-java
+ org.xerial.snappy
+ snappy-java
-
-
${project.groupId}
pulsar-functions-worker
${project.version}
-
${project.groupId}
pulsar-functions-local-runner-original
${project.version}
test
-
${project.groupId}
pulsar-client-messagecrypto-bc
${project.version}
test
-
org.awaitility
awaitility
test
-
-
org.eclipse.jetty
jetty-server
-
org.eclipse.jetty
jetty-alpn-conscrypt-server
-
org.eclipse.jetty
jetty-servlet
-
org.eclipse.jetty
jetty-servlets
-
org.glassfish.jersey.core
jersey-server
-
org.glassfish.jersey.containers
jersey-container-servlet-core
-
org.glassfish.jersey.containers
jersey-container-servlet
-
org.glassfish.jersey.media
jersey-media-json-jackson
-
org.glassfish.jersey.test-framework
jersey-test-framework-core
test
${jersey.version}
-
org.glassfish.jersey.test-framework.providers
jersey-test-framework-provider-grizzly2
test
${jersey.version}
-
jakarta.activation
jakarta.activation-api
-
com.fasterxml.jackson.jaxrs
jackson-jaxrs-json-provider
-
org.glassfish.jersey.inject
jersey-hk2
-
com.fasterxml.jackson.module
jackson-module-jsonSchema
-
org.slf4j
jul-to-slf4j
-
org.slf4j
jcl-over-slf4j
-
com.google.guava
guava
-
com.beust
jcommander
-
io.swagger
swagger-annotations
-
io.prometheus
simpleclient
-
io.prometheus
simpleclient_jetty
-
io.prometheus
simpleclient_hotspot
-
io.prometheus
simpleclient_caffeine
-
io.swagger
swagger-core
-
org.hdrhistogram
HdrHistogram
-
com.google.code.gson
gson
-
com.github.zafarkhaja
java-semver
-
org.apache.avro
avro
${avro.version}
-
com.carrotsearch
hppc
-
org.roaringbitmap
RoaringBitmap
-
${project.groupId}
pulsar-functions-api-examples
@@ -366,7 +307,6 @@
pom
test
-
${project.groupId}
pulsar-io-batch-data-generator
@@ -374,7 +314,6 @@
pom
test
-
${project.groupId}
pulsar-io-data-generator
@@ -382,7 +321,6 @@
pom
test
-
${project.groupId}
pulsar-metadata
@@ -390,7 +328,6 @@
test-jar
test
-
javax.xml.bind
jaxb-api
@@ -401,35 +338,28 @@
-
com.sun.activation
javax.activation
-
-
${project.groupId}
pulsar-transaction-coordinator
${project.version}
-
-
${project.groupId}
pulsar-package-core
${project.version}
-
io.etcd
jetcd-test
test
-
@@ -462,7 +392,6 @@
-
maven-dependency-plugin
@@ -506,7 +435,6 @@
-
org.apache.maven.plugins
maven-surefire-plugin
@@ -521,7 +449,6 @@
-
org.xolstice.maven.plugins
protobuf-maven-plugin
@@ -618,7 +545,6 @@
-
@@ -637,7 +563,6 @@
test-jar
test
-
${project.groupId}
pulsar-package-core
@@ -645,7 +570,6 @@
test-jar
test
-
${project.groupId}
pulsar-metadata
@@ -793,7 +717,7 @@
-
+
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index da37d1b7e545c..f4e5caacded63 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -2208,9 +2208,10 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su
internalCreateSubscriptionForNonPartitionedTopic(asyncResponse,
subscriptionName, targetMessageId, authoritative, replicated, properties);
} else {
- boolean allowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName);
- getPartitionedTopicMetadataAsync(topicName,
- authoritative, allowAutoTopicCreation).thenAccept(partitionMetadata -> {
+ pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName)
+ .thenCompose(allowAutoTopicCreation ->
+ getPartitionedTopicMetadataAsync(topicName, authoritative, allowAutoTopicCreation))
+ .thenAccept(partitionMetadata -> {
final int numPartitions = partitionMetadata.partitions;
if (numPartitions > 0) {
final CompletableFuture future = new CompletableFuture<>();
@@ -2306,13 +2307,13 @@ private void internalCreateSubscriptionForNonPartitionedTopic(
MessageIdImpl targetMessageId, boolean authoritative, boolean replicated,
Map properties) {
- boolean isAllowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName);
-
- validateTopicOwnershipAsync(topicName, authoritative)
- .thenCompose(__ -> {
- validateTopicOperation(topicName, TopicOperation.SUBSCRIBE, subscriptionName);
- return pulsar().getBrokerService().getTopic(topicName.toString(), isAllowAutoTopicCreation);
- }).thenApply(optTopic -> {
+ pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName)
+ .thenCompose(isAllowAutoTopicCreation -> validateTopicOwnershipAsync(topicName, authoritative)
+ .thenCompose(__ -> {
+ validateTopicOperation(topicName, TopicOperation.SUBSCRIBE, subscriptionName);
+ return pulsar().getBrokerService().getTopic(topicName.toString(), isAllowAutoTopicCreation);
+ }))
+ .thenApply(optTopic -> {
if (optTopic.isPresent()) {
return optTopic.get();
} else {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
index e0127bd38646f..26c31a6e8a955 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java
@@ -554,4 +554,10 @@ public static void filterBrokersWithLargeTopicCount(Set brokerCandidateC
brokerCandidateCache.addAll(filteredBrokerCandidates);
}
}
+
+ public static NamespaceBundle getNamespaceBundle(PulsarService pulsar, String bundle) {
+ final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+ final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
+ return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange);
+ }
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
index 074083052895e..301f19213c352 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.broker.loadbalance.impl;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
@@ -642,9 +643,21 @@ public synchronized void doLoadShedding() {
if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) {
return;
}
+ NamespaceBundle bundleToUnload = LoadManagerShared.getNamespaceBundle(pulsar, bundle);
+ Optional destBroker = this.selectBroker(bundleToUnload);
+ if (!destBroker.isPresent()) {
+ log.info("[{}] No broker available to unload bundle {} from broker {}",
+ strategy.getClass().getSimpleName(), bundle, broker);
+ return;
+ }
+ if (destBroker.get().equals(broker)) {
+ log.warn("[{}] The destination broker {} is the same as the current owner broker for Bundle {}",
+ strategy.getClass().getSimpleName(), destBroker.get(), bundle);
+ return;
+ }
- log.info("[{}] Unloading bundle: {} from broker {}",
- strategy.getClass().getSimpleName(), bundle, broker);
+ log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}",
+ strategy.getClass().getSimpleName(), bundle, broker, destBroker.get());
try {
pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange);
loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis());
@@ -799,16 +812,56 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni
// If the given bundle is already in preallocated, return the selected broker.
return Optional.of(preallocatedBundleToBroker.get(bundle));
}
- final BundleData data = loadData.getBundleData().computeIfAbsent(bundle,
- key -> getBundleDataOrDefault(bundle));
- brokerCandidateCache.clear();
- LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
- getAvailableBrokers(),
- brokerTopicLoadingPredicate);
- // filter brokers which owns topic higher than threshold
- LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData,
- conf.getLoadBalancerBrokerMaxTopics());
+ Optional broker = selectBroker(serviceUnit);
+ if (!broker.isPresent()) {
+ // If no broker is selected, return empty.
+ return broker;
+ }
+ // Add new bundle to preallocated.
+ preallocateBundle(bundle, broker.get());
+ return broker;
+ }
+ } finally {
+ selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ }
+ }
+
+ private void preallocateBundle(String bundle, String broker) {
+ final BundleData data = loadData.getBundleData().computeIfAbsent(bundle,
+ key -> getBundleDataOrDefault(bundle));
+ loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data);
+ preallocatedBundleToBroker.put(bundle, broker);
+
+ final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
+ final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
+ final ConcurrentOpenHashMap> namespaceToBundleRange =
+ brokerToNamespaceToBundleRange
+ .computeIfAbsent(broker,
+ k -> ConcurrentOpenHashMap.>newBuilder()
+ .build());
+ synchronized (namespaceToBundleRange) {
+ namespaceToBundleRange.computeIfAbsent(namespaceName,
+ k -> ConcurrentOpenHashSet.newBuilder().build())
+ .add(bundleRange);
+ }
+ }
+
+ @VisibleForTesting
+ Optional selectBroker(final ServiceUnitId serviceUnit) {
+ synchronized (brokerCandidateCache) {
+ final String bundle = serviceUnit.toString();
+ final BundleData data = loadData.getBundleData().computeIfAbsent(bundle,
+ key -> getBundleDataOrDefault(bundle));
+ brokerCandidateCache.clear();
+ LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
+ getAvailableBrokers(),
+ brokerTopicLoadingPredicate);
+
+ // filter brokers which owns topic higher than threshold
+ LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData,
+ conf.getLoadBalancerBrokerMaxTopics());
// distribute namespaces to domain and brokers according to anti-affinity-group
LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, serviceUnit.toString(),
@@ -820,71 +873,50 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni
brokerToNamespaceToBundleRange);
log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle);
- // Use the filter pipeline to finalize broker candidates.
- try {
- for (BrokerFilter filter : filterPipeline) {
- filter.filter(brokerCandidateCache, data, loadData, conf);
- }
- } catch (BrokerFilterException x) {
- // restore the list of brokers to the full set
- LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
- getAvailableBrokers(),
- brokerTopicLoadingPredicate);
- }
-
- if (brokerCandidateCache.isEmpty()) {
- // restore the list of brokers to the full set
- LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
- getAvailableBrokers(),
- brokerTopicLoadingPredicate);
+ // Use the filter pipeline to finalize broker candidates.
+ try {
+ for (BrokerFilter filter : filterPipeline) {
+ filter.filter(brokerCandidateCache, data, loadData, conf);
}
+ } catch (BrokerFilterException x) {
+ // restore the list of brokers to the full set
+ LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
+ getAvailableBrokers(),
+ brokerTopicLoadingPredicate);
+ }
- // Choose a broker among the potentially smaller filtered list, when possible
- Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
- if (log.isDebugEnabled()) {
- log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache);
- }
+ if (brokerCandidateCache.isEmpty()) {
+ // restore the list of brokers to the full set
+ LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
+ getAvailableBrokers(),
+ brokerTopicLoadingPredicate);
+ }
- if (!broker.isPresent()) {
- // No brokers available
- return broker;
- }
+ // Choose a broker among the potentially smaller filtered list, when possible
+ Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
+ if (log.isDebugEnabled()) {
+ log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache);
+ }
- final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
- final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage();
- if (maxUsage > overloadThreshold) {
- // All brokers that were in the filtered list were overloaded, so check if there is a better broker
- LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
- getAvailableBrokers(),
- brokerTopicLoadingPredicate);
- Optional brokerTmp =
- placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
- if (brokerTmp.isPresent()) {
- broker = brokerTmp;
- }
- }
+ if (!broker.isPresent()) {
+ // No brokers available
+ return broker;
+ }
- // Add new bundle to preallocated.
- loadData.getBrokerData().get(broker.get()).getPreallocatedBundleData().put(bundle, data);
- preallocatedBundleToBroker.put(bundle, broker.get());
-
- final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle);
- final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle);
- final ConcurrentOpenHashMap> namespaceToBundleRange =
- brokerToNamespaceToBundleRange
- .computeIfAbsent(broker.get(),
- k -> ConcurrentOpenHashMap.>newBuilder()
- .build());
- synchronized (namespaceToBundleRange) {
- namespaceToBundleRange.computeIfAbsent(namespaceName,
- k -> ConcurrentOpenHashSet.newBuilder().build())
- .add(bundleRange);
+ final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
+ final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage();
+ if (maxUsage > overloadThreshold) {
+ // All brokers that were in the filtered list were overloaded, so check if there is a better broker
+ LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache,
+ getAvailableBrokers(),
+ brokerTopicLoadingPredicate);
+ Optional brokerTmp =
+ placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf);
+ if (brokerTmp.isPresent()) {
+ broker = brokerTmp;
}
- return broker;
}
- } finally {
- selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ return broker;
}
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
index ae1d2a5bab02b..7baebc7d1b6d4 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java
@@ -83,9 +83,11 @@ protected void internalLookupTopicAsync(TopicName topicName, boolean authoritati
// Currently, it's hard to check the non-persistent-non-partitioned topic, because it only exists in the broker,
// it doesn't have metadata. If the topic is non-persistent and non-partitioned, we'll return the true flag.
- CompletableFuture existFuture = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName)
- || (!topicName.isPersistent() && !topicName.isPartitioned())
- ? CompletableFuture.completedFuture(true) : pulsar().getNamespaceService().checkTopicExists(topicName);
+ CompletableFuture existFuture = pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName)
+ .thenCompose(isAllowAutoTopicCreation ->
+ isAllowAutoTopicCreation || (!topicName.isPersistent() && !topicName.isPartitioned())
+ ? CompletableFuture.completedFuture(true) :
+ pulsar().getNamespaceService().checkTopicExists(topicName));
existFuture.thenAccept(exist -> {
if (!exist) {
completeLookupResponseExceptionally(asyncResponse, new RestException(Response.Status.NOT_FOUND,
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
index 736fcf1f5e1e5..76e1df0d93d7f 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.broker.service;
import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Lists;
@@ -40,6 +41,7 @@
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.ToLongFunction;
+import javax.annotation.Nonnull;
import lombok.Getter;
import org.apache.bookkeeper.mledger.util.StatsBuckets;
import org.apache.commons.collections4.CollectionUtils;
@@ -57,6 +59,7 @@
import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException;
import org.apache.pulsar.broker.stats.prometheus.metrics.Summary;
import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.BacklogQuota;
import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies;
@@ -131,7 +134,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener USER_CREATED_PRODUCER_COUNTER_UPDATER =
AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount");
- private volatile int userCreatedProducerCount = 0;
+ protected volatile int userCreatedProducerCount = 0;
protected volatile Optional topicEpoch = Optional.empty();
private volatile boolean hasExclusiveProducer;
@@ -1065,6 +1068,12 @@ public PublishRateLimiter getBrokerPublishRateLimiter() {
return brokerService.getBrokerPublishRateLimiter();
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and we can use
+ * #{@link AbstractTopic#updateResourceGroupLimiter(Policies)} to instead of it.
+ */
+ @Deprecated
public void updateResourceGroupLimiter(Optional optPolicies) {
Policies policies;
try {
@@ -1078,17 +1087,20 @@ public void updateResourceGroupLimiter(Optional optPolicies) {
log.warn("[{}] Error getting policies {} and publish throttling will be disabled", topic, e.getMessage());
policies = new Policies();
}
+ updateResourceGroupLimiter(policies);
+ }
+ public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) {
+ requireNonNull(namespacePolicies);
// attach the resource-group level rate limiters, if set
- String rgName = policies.resource_group_name;
+ String rgName = namespacePolicies.resource_group_name;
if (rgName != null) {
final ResourceGroup resourceGroup =
- brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName);
+ brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName);
if (resourceGroup != null) {
this.resourceGroupRateLimitingEnabled = true;
this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter();
- this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(),
- () -> this.enableCnxAutoRead());
+ this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), this::enableCnxAutoRead);
log.info("Using resource group {} rate limiter for topic {}", rgName, topic);
return;
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index cac70f6896e5f..08550886ecb4a 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
import static org.apache.commons.collections.CollectionUtils.isEmpty;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
@@ -56,6 +57,7 @@
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
@@ -72,6 +74,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Predicate;
+import javax.annotation.Nonnull;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -931,7 +934,15 @@ public CompletableFuture> getTopicIfExists(final String topic) {
}
public CompletableFuture getOrCreateTopic(final String topic) {
- return getTopic(topic, isAllowAutoTopicCreation(topic)).thenApply(Optional::get);
+ final TopicName topicName;
+ try {
+ topicName = TopicName.get(topic);
+ } catch (Throwable ex) {
+ return FutureUtil.failedFuture(ex);
+ }
+ return isAllowAutoTopicCreationAsync(topicName)
+ .thenCompose(isAllowAutoTopicCreation -> getTopic(topic, isAllowAutoTopicCreation)
+ .thenApply(Optional::get));
}
public CompletableFuture> getTopic(final String topic, boolean createIfMissing) {
@@ -2704,12 +2715,9 @@ public CompletableFuture fetchPartitionedTopicMetadata
if (pulsar.getNamespaceService() == null) {
return FutureUtil.failedFuture(new NamingException("namespace service is not ready"));
}
- Optional policies =
- pulsar.getPulsarResources().getNamespaceResources()
- .getPoliciesIfCached(topicName.getNamespaceObject());
- return pulsar.getNamespaceService().checkTopicExists(topicName)
- .thenCompose(topicExists -> {
- return fetchPartitionedTopicMetadataAsync(topicName)
+ return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject())
+ .thenCompose(policies -> pulsar.getNamespaceService().checkTopicExists(topicName)
+ .thenCompose(topicExists -> fetchPartitionedTopicMetadataAsync(topicName)
.thenCompose(metadata -> {
CompletableFuture future = new CompletableFuture<>();
@@ -2721,7 +2729,8 @@ public CompletableFuture fetchPartitionedTopicMetadata
if (metadata.partitions == 0
&& !topicExists
&& !topicName.isPartitioned()
- && pulsar.getBrokerService().isAllowAutoTopicCreation(topicName, policies)
+ && pulsar.getBrokerService()
+ .isAllowAutoTopicCreation(topicName, policies)
&& pulsar.getBrokerService()
.isDefaultTopicTypePartitioned(topicName, policies)) {
@@ -2729,10 +2738,13 @@ public CompletableFuture fetchPartitionedTopicMetadata
.createDefaultPartitionedTopicAsync(topicName, policies)
.thenAccept(md -> future.complete(md))
.exceptionally(ex -> {
+ log.info("[{}] The partitioned topic is already"
+ + " created, try to refresh the cache and read"
+ + " again.", topicName);
if (ex.getCause()
instanceof MetadataStoreException.AlreadyExistsException) {
// The partitioned topic might be created concurrently
- fetchPartitionedTopicMetadataAsync(topicName)
+ fetchPartitionedTopicMetadataAsync(topicName, true)
.whenComplete((metadata2, ex2) -> {
if (ex2 == null) {
future.complete(metadata2);
@@ -2741,6 +2753,9 @@ public CompletableFuture fetchPartitionedTopicMetadata
}
});
} else {
+ log.error("[{}] operation of creating partitioned"
+ + " topic metadata failed",
+ topicName, ex);
future.completeExceptionally(ex);
}
return null;
@@ -2751,8 +2766,8 @@ public CompletableFuture fetchPartitionedTopicMetadata
});
return future;
- });
- });
+ }))
+ );
}
@SuppressWarnings("deprecation")
@@ -2780,9 +2795,14 @@ private CompletableFuture createDefaultPartitionedTopi
}
public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName) {
+ return fetchPartitionedTopicMetadataAsync(topicName, false);
+ }
+
+ public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName,
+ boolean refreshCacheAndGet) {
// gets the number of partitions from the configuration cache
return pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources()
- .getPartitionedTopicMetadataAsync(topicName).thenApply(metadata -> {
+ .getPartitionedTopicMetadataAsync(topicName, refreshCacheAndGet).thenApply(metadata -> {
// if the partitioned topic is not found in metadata, then the topic is not partitioned
return metadata.orElseGet(() -> new PartitionedTopicMetadata());
});
@@ -2947,11 +2967,23 @@ private void foreachCnx(Consumer consumer) {
cnxSet.forEach(consumer);
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)}
+ * You can use #{@link BrokerService#isAllowAutoTopicCreationAsync(TopicName)}
+ */
+ @Deprecated
public boolean isAllowAutoTopicCreation(final String topic) {
TopicName topicName = TopicName.get(topic);
return isAllowAutoTopicCreation(topicName);
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)}
+ * You can use #{@link BrokerService#isAllowAutoTopicCreationAsync(TopicName)}
+ */
+ @Deprecated
public boolean isAllowAutoTopicCreation(final TopicName topicName) {
Optional policies =
pulsar.getPulsarResources().getNamespaceResources()
@@ -2959,6 +2991,11 @@ public boolean isAllowAutoTopicCreation(final TopicName topicName) {
return isAllowAutoTopicCreation(topicName, policies);
}
+ public CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName) {
+ return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject())
+ .thenApply(policies -> isAllowAutoTopicCreation(topicName, policies));
+ }
+
public boolean isAllowAutoTopicCreation(final TopicName topicName, final Optional policies) {
if (policies.isPresent() && policies.get().deleted) {
log.info("Preventing AutoTopicCreation on a namespace that is being deleted {}",
@@ -3005,11 +3042,23 @@ private AutoTopicCreationOverride getAutoTopicCreationOverride(final TopicName t
return null;
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it.
+ */
+ @Deprecated
public boolean isAllowAutoSubscriptionCreation(final String topic) {
TopicName topicName = TopicName.get(topic);
return isAllowAutoSubscriptionCreation(topicName);
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it.
+ */
+ @Deprecated
public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) {
AutoSubscriptionCreationOverride autoSubscriptionCreationOverride =
getAutoSubscriptionCreationOverride(topicName);
@@ -3020,6 +3069,12 @@ public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) {
}
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it.
+ */
+ @Deprecated
private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(final TopicName topicName) {
Optional policies =
pulsar.getPulsarResources().getNamespaceResources().getPoliciesIfCached(topicName.getNamespaceObject());
@@ -3031,6 +3086,19 @@ private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(fin
return null;
}
+ public @Nonnull CompletionStage isAllowAutoSubscriptionCreationAsync(@Nonnull TopicName tpName) {
+ requireNonNull(tpName);
+ // namespace level policies
+ return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(tpName.getNamespaceObject())
+ .thenApply(policies -> {
+ if (policies.isPresent() && policies.get().autoSubscriptionCreationOverride != null) {
+ return policies.get().autoSubscriptionCreationOverride.isAllowAutoSubscriptionCreation();
+ }
+ // broker level policies
+ return pulsar.getConfiguration().isAllowAutoSubscriptionCreation();
+ });
+ }
+
public boolean isSystemTopic(String topic) {
return isSystemTopic(TopicName.get(topic));
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index f2cd77e485e23..8924b750eb624 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -318,6 +318,12 @@ public Future sendMessages(final List entries, EntryBatchSizes batc
msgOutCounter.add(totalMessages);
bytesOutCounter.add(totalBytes);
chunkedMessageRate.recordMultipleEvents(totalChunkedMessages, 0);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}-{}] Sent messages to client fail by IO exception[{}], close the connection"
+ + " immediately. Consumer: {}", topicName, subscription,
+ status.cause() == null ? "" : status.cause().getMessage(), this.toString());
+ }
}
});
return writeAndFlushPromise;
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
index c1045c3d14b73..ef7cfdf14d87d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java
@@ -53,6 +53,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionStage;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -994,7 +995,8 @@ protected void handleSubscribe(final CommandSubscribe subscribe) {
remoteAddress, getPrincipal());
}
- log.info("[{}] Subscribing on topic {} / {}", remoteAddress, topicName, subscriptionName);
+ log.info("[{}] Subscribing on topic {} / {}. consumerId: {}", this.ctx().channel().toString(),
+ topicName, subscriptionName, consumerId);
try {
Metadata.validateMetadata(metadata,
service.getPulsar().getConfiguration().getMaxConsumerMetadataSize());
@@ -1033,48 +1035,53 @@ protected void handleSubscribe(final CommandSubscribe subscribe) {
return null;
}
- boolean createTopicIfDoesNotExist = forceTopicCreation
- && service.isAllowAutoTopicCreation(topicName.toString());
-
- service.getTopic(topicName.toString(), createTopicIfDoesNotExist)
+ service.isAllowAutoTopicCreationAsync(topicName)
+ .thenCompose(isAllowAutoTopicCreation -> {
+ final boolean createTopicIfDoesNotExist = forceTopicCreation && isAllowAutoTopicCreation;
+ return service.getTopic(topicName.toString(), createTopicIfDoesNotExist);
+ })
.thenCompose(optTopic -> {
if (!optTopic.isPresent()) {
return FutureUtil
.failedFuture(new TopicNotFoundException(
"Topic " + topicName + " does not exist"));
}
-
- Topic topic = optTopic.get();
-
- boolean rejectSubscriptionIfDoesNotExist = isDurable
- && !service.isAllowAutoSubscriptionCreation(topicName.toString())
- && !topic.getSubscriptions().containsKey(subscriptionName)
- && topic.isPersistent();
-
- if (rejectSubscriptionIfDoesNotExist) {
- return FutureUtil
- .failedFuture(
- new SubscriptionNotFoundException(
- "Subscription does not exist"));
- }
-
- SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this)
- .subscriptionName(subscriptionName)
- .consumerId(consumerId).subType(subType).priorityLevel(priorityLevel)
- .consumerName(consumerName).isDurable(isDurable)
- .startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted)
- .initialPosition(initialPosition)
- .startMessageRollbackDurationSec(startMessageRollbackDurationSec)
- .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta)
- .subscriptionProperties(subscriptionProperties)
- .consumerEpoch(consumerEpoch)
- .build();
- if (schema != null) {
- return topic.addSchemaIfIdleOrCheckCompatible(schema)
- .thenCompose(v -> topic.subscribe(option));
- } else {
- return topic.subscribe(option);
- }
+ final Topic topic = optTopic.get();
+ return service.isAllowAutoSubscriptionCreationAsync(topicName)
+ .thenCompose(isAllowAutoSubscriptionCreation -> {
+ boolean rejectSubscriptionIfDoesNotExist = isDurable
+ && !isAllowAutoSubscriptionCreation
+ && !topic.getSubscriptions().containsKey(subscriptionName)
+ && topic.isPersistent();
+
+ if (rejectSubscriptionIfDoesNotExist) {
+ return FutureUtil
+ .failedFuture(
+ new SubscriptionNotFoundException(
+ "Subscription does not exist"));
+ }
+
+ SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this)
+ .subscriptionName(subscriptionName)
+ .consumerId(consumerId).subType(subType)
+ .priorityLevel(priorityLevel)
+ .consumerName(consumerName).isDurable(isDurable)
+ .startMessageId(startMessageId).metadata(metadata)
+ .readCompacted(readCompacted)
+ .initialPosition(initialPosition)
+ .startMessageRollbackDurationSec(startMessageRollbackDurationSec)
+ .replicatedSubscriptionStateArg(isReplicated)
+ .keySharedMeta(keySharedMeta)
+ .subscriptionProperties(subscriptionProperties)
+ .consumerEpoch(consumerEpoch)
+ .build();
+ if (schema != null) {
+ return topic.addSchemaIfIdleOrCheckCompatible(schema)
+ .thenCompose(v -> topic.subscribe(option));
+ } else {
+ return topic.subscribe(option);
+ }
+ });
})
.thenAccept(consumer -> {
if (consumerFuture.complete(consumer)) {
@@ -1285,33 +1292,39 @@ protected void handleProducer(final CommandProducer cmdProducer) {
schemaVersionFuture.thenAccept(schemaVersion -> {
topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> {
- CompletableFuture createInitSubFuture;
+ CompletionStage createInitSubFuture;
if (!Strings.isNullOrEmpty(initialSubscriptionName)
&& topic.isPersistent()
&& !topic.getSubscriptions().containsKey(initialSubscriptionName)) {
- if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) {
- String msg =
- "Could not create the initial subscription due to the auto subscription "
- + "creation is not allowed.";
- if (producerFuture.completeExceptionally(
- new BrokerServiceException.NotAllowedException(msg))) {
- log.warn("[{}] {} initialSubscriptionName: {}, topic: {}",
- remoteAddress, msg, initialSubscriptionName, topicName);
- commandSender.sendErrorResponse(requestId,
- ServerError.NotAllowedError, msg);
- }
- producers.remove(producerId, producerFuture);
- return;
- }
- createInitSubFuture =
- topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest,
- false, null);
+ createInitSubFuture = service.isAllowAutoSubscriptionCreationAsync(topicName)
+ .thenCompose(isAllowAutoSubscriptionCreation -> {
+ if (!isAllowAutoSubscriptionCreation) {
+ return FutureUtil.failedFuture(
+ new BrokerServiceException.NotAllowedException(
+ "Could not create the initial subscription due to"
+ + " the auto subscription creation is not allowed."));
+ }
+ return topic.createSubscription(initialSubscriptionName,
+ InitialPosition.Earliest, false, null);
+ });
} else {
createInitSubFuture = CompletableFuture.completedFuture(null);
}
createInitSubFuture.whenComplete((sub, ex) -> {
if (ex != null) {
+ final Throwable rc = FutureUtil.unwrapCompletionException(ex);
+ if (rc instanceof BrokerServiceException.NotAllowedException) {
+ log.warn("[{}] {} initialSubscriptionName: {}, topic: {}",
+ remoteAddress, rc.getMessage(), initialSubscriptionName, topicName);
+ if (producerFuture.completeExceptionally(rc)) {
+ commandSender.sendErrorResponse(requestId,
+ ServerError.NotAllowedError, rc.getMessage());
+ }
+ producers.remove(producerId, producerFuture);
+ return;
+ }
+
String msg =
"Failed to create the initial subscription: " + ex.getCause().getMessage();
log.warn("[{}] {} initialSubscriptionName: {}, topic: {}",
@@ -1537,11 +1550,12 @@ private void printSendCommandDebug(CommandSend send, ByteBuf headersAndPayload)
headersAndPayload.resetReaderIndex();
if (log.isDebugEnabled()) {
log.debug("[{}] Received send message request. producer: {}:{} {}:{} size: {},"
- + " partition key is: {}, ordering key is {}",
+ + " partition key is: {}, ordering key is {}, uncompressedSize is {}",
remoteAddress, send.getProducerId(), send.getSequenceId(), msgMetadata.getProducerName(),
msgMetadata.getSequenceId(), headersAndPayload.readableBytes(),
msgMetadata.hasPartitionKey() ? msgMetadata.getPartitionKey() : null,
- msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null);
+ msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null,
+ msgMetadata.getUncompressedSize());
}
}
@@ -1573,6 +1587,12 @@ protected void handleAck(CommandAck ack) {
}
return null;
});
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("Consumer future is not complete(not complete or error), but received command ack. so discard"
+ + " this command. consumerId: {}, cnx: {}, messageIdCount: {}", ack.getConsumerId(),
+ this.ctx().channel().toString(), ack.getMessageIdsCount());
+ }
}
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java
index c171fea0a6f97..419c6d88b22a1 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java
@@ -30,6 +30,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nonnull;
+import org.apache.commons.lang3.tuple.MutablePair;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener;
@@ -220,12 +221,25 @@ public TopicPolicies getTopicPolicies(TopicName topicName,
NamespaceName namespace = topicName.getNamespaceObject();
prepareInitPoliciesCache(namespace, new CompletableFuture<>());
}
- if (policyCacheInitMap.containsKey(topicName.getNamespaceObject())
- && !policyCacheInitMap.get(topicName.getNamespaceObject())) {
- throw new TopicPoliciesCacheNotInitException();
+
+ MutablePair result = new MutablePair<>();
+ policyCacheInitMap.compute(topicName.getNamespaceObject(), (k, initialized) -> {
+ if (initialized == null || !initialized) {
+ result.setLeft(new TopicPoliciesCacheNotInitException());
+ } else {
+ TopicPolicies topicPolicies =
+ isGlobal ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName()))
+ : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName()));
+ result.setRight(topicPolicies);
+ }
+ return initialized;
+ });
+
+ if (result.getLeft() != null) {
+ throw result.getLeft();
+ } else {
+ return result.getRight();
}
- return isGlobal ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName()))
- : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName()));
}
@Override
@@ -389,7 +403,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp
private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) {
CompletableFuture> readerFuture = readerCaches.remove(namespace);
- policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace));
+
if (cleanOwnedBundlesCount) {
ownedBundlesCountPerNamespace.remove(namespace);
}
@@ -400,7 +414,11 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean
return null;
});
}
- policyCacheInitMap.remove(namespace);
+
+ policyCacheInitMap.compute(namespace, (k, v) -> {
+ policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace));
+ return null;
+ });
}
private void readMorePolicies(SystemTopicClient.Reader reader) {
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
index 54f738101be27..a021e7ce94def 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java
@@ -157,18 +157,20 @@ public CompletableFuture initialize() {
return brokerService.pulsar().getPulsarResources().getNamespaceResources()
.getPoliciesAsync(TopicName.get(topic).getNamespaceObject())
.thenAccept(optPolicies -> {
+ final Policies policies;
if (!optPolicies.isPresent()) {
log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic);
isEncryptionRequired = false;
+ policies = new Policies();
} else {
- Policies policies = optPolicies.get();
+ policies = optPolicies.get();
updateTopicPolicyByNamespacePolicy(policies);
isEncryptionRequired = policies.encryption_required;
isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema;
schemaValidationEnforced = policies.schema_validation_enforced;
}
updatePublishDispatcher();
- updateResourceGroupLimiter(optPolicies);
+ updateResourceGroupLimiter(policies);
});
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
index cd7f5e9ea64c4..568ae0915cb3d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java
@@ -364,6 +364,12 @@ public static CompletableFuture> getPoliciesAsync(BrokerServi
return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace);
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call. we can use #{@link DispatchRateLimiter#getPoliciesAsync(BrokerService, String)} to instead of it.
+ */
+ @Deprecated
public static Optional getPolicies(BrokerService brokerService, String topicName) {
final NamespaceName namespace = TopicName.get(topicName).getNamespaceObject();
return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesIfCached(namespace);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
index e40e7585f1637..4c3d8e1a467c5 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java
@@ -55,6 +55,8 @@ public class MessageDeduplication {
private final ManagedLedger managedLedger;
private ManagedCursor managedCursor;
+ private static final String IS_LAST_CHUNK = "isLastChunk";
+
enum Status {
// Deduplication is initialized
@@ -324,11 +326,12 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade
String producerName = publishContext.getProducerName();
long sequenceId = publishContext.getSequenceId();
long highestSequenceId = Math.max(publishContext.getHighestSequenceId(), sequenceId);
+ MessageMetadata md = null;
if (producerName.startsWith(replicatorPrefix)) {
// Message is coming from replication, we need to use the original producer name and sequence id
// for the purpose of deduplication and not rely on the "replicator" name.
int readerIndex = headersAndPayload.readerIndex();
- MessageMetadata md = Commands.parseMessageMetadata(headersAndPayload);
+ md = Commands.parseMessageMetadata(headersAndPayload);
producerName = md.getProducerName();
sequenceId = md.getSequenceId();
highestSequenceId = Math.max(md.getHighestSequenceId(), sequenceId);
@@ -337,7 +340,23 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade
publishContext.setOriginalHighestSequenceId(highestSequenceId);
headersAndPayload.readerIndex(readerIndex);
}
-
+ long chunkID = -1;
+ long totalChunk = -1;
+ if (publishContext.isChunked()) {
+ if (md == null) {
+ int readerIndex = headersAndPayload.readerIndex();
+ md = Commands.parseMessageMetadata(headersAndPayload);
+ headersAndPayload.readerIndex(readerIndex);
+ }
+ chunkID = md.getChunkId();
+ totalChunk = md.getNumChunksFromMsg();
+ }
+ // All chunks of a message use the same message metadata and sequence ID,
+ // so we only need to check the sequence ID for the last chunk in a chunk message.
+ if (chunkID != -1 && chunkID != totalChunk - 1) {
+ publishContext.setProperty(IS_LAST_CHUNK, Boolean.FALSE);
+ return MessageDupStatus.NotDup;
+ }
// Synchronize the get() and subsequent put() on the map. This would only be relevant if the producer
// disconnects and re-connects very quickly. At that point the call can be coming from a different thread
synchronized (highestSequencedPushed) {
@@ -363,6 +382,11 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade
}
highestSequencedPushed.put(producerName, highestSequenceId);
}
+ // Only put sequence ID into highestSequencedPushed and
+ // highestSequencedPersisted until receive and persistent the last chunk.
+ if (chunkID != -1 && chunkID == totalChunk - 1) {
+ publishContext.setProperty(IS_LAST_CHUNK, Boolean.TRUE);
+ }
return MessageDupStatus.NotDup;
}
@@ -383,8 +407,10 @@ public void recordMessagePersisted(PublishContext publishContext, PositionImpl p
sequenceId = publishContext.getOriginalSequenceId();
highestSequenceId = publishContext.getOriginalHighestSequenceId();
}
-
- highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId));
+ Boolean isLastChunk = (Boolean) publishContext.getProperty(IS_LAST_CHUNK);
+ if (isLastChunk == null || isLastChunk) {
+ highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId));
+ }
if (++snapshotCounter >= snapshotInterval) {
snapshotCounter = 0;
takeSnapshot(position);
@@ -482,6 +508,10 @@ public long getLastPublishedSequenceId(String producerName) {
}
public void takeSnapshot() {
+ if (!isEnabled()) {
+ return;
+ }
+
Integer interval = topic.getHierarchyTopicPolicies().getDeduplicationSnapshotIntervalSeconds().get();
long currentTimeStamp = System.currentTimeMillis();
if (interval == null || interval <= 0
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
index 5054b039af79b..14074fc280bd6 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.broker.service.persistent;
import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled;
import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames;
@@ -49,6 +50,7 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
import lombok.Getter;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -316,7 +318,7 @@ public CompletableFuture initialize() {
if (!optPolicies.isPresent()) {
isEncryptionRequired = false;
updatePublishDispatcher();
- updateResourceGroupLimiter(optPolicies);
+ updateResourceGroupLimiter(new Policies());
initializeDispatchRateLimiterIfNeeded();
updateSubscribeRateLimiter();
return;
@@ -332,7 +334,7 @@ public CompletableFuture initialize() {
updatePublishDispatcher();
- updateResourceGroupLimiter(optPolicies);
+ updateResourceGroupLimiter(policies);
this.isEncryptionRequired = policies.encryption_required;
@@ -2419,7 +2421,8 @@ public void updateDispatchRateLimiter() {
}
@Override
- public CompletableFuture onPoliciesUpdate(Policies data) {
+ public CompletableFuture onPoliciesUpdate(@Nonnull Policies data) {
+ requireNonNull(data);
if (log.isDebugEnabled()) {
log.debug("[{}] isEncryptionRequired changes: {} -> {}", topic, isEncryptionRequired,
data.encryption_required);
@@ -2443,7 +2446,7 @@ public CompletableFuture onPoliciesUpdate(Policies data) {
updatePublishDispatcher();
- this.updateResourceGroupLimiter(Optional.of(data));
+ updateResourceGroupLimiter(data);
List> producerCheckFutures = new ArrayList<>(producers.size());
producers.values().forEach(producer -> producerCheckFutures.add(
@@ -2783,6 +2786,11 @@ public synchronized void triggerCompaction()
throws PulsarServerException, AlreadyRunningException {
if (currentCompaction.isDone()) {
currentCompaction = brokerService.pulsar().getCompactor().compact(topic);
+ currentCompaction.whenComplete((ignore, ex) -> {
+ if (ex != null){
+ log.warn("[{}] Compaction failure.", topic, ex);
+ }
+ });
} else {
throw new AlreadyRunningException("Compaction already in progress");
}
@@ -2860,7 +2868,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem
.mapToInt(subscription -> subscription.getConsumers().size())
.sum();
if (hasSchema
- || (!producers.isEmpty())
+ || (userCreatedProducerCount > 0)
|| (numActiveConsumers != 0)
|| (ledger.getTotalSize() != 0)) {
return checkSchemaCompatibleForConsumer(schema);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
index 1e1245ed36b6f..cf1603788f1fd 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java
@@ -192,10 +192,14 @@ private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) {
sub.acknowledgeMessage(Collections.singletonList(pos), AckType.Cumulative, Collections.emptyMap());
} else {
// Subscription doesn't exist. We need to force the creation of the subscription in this cluster, because
- log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subcription",
+ log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subscription",
topic, update.getSubscriptionName(), updatedMessageId.getLedgerId(), pos);
- topic.createSubscription(update.getSubscriptionName(),
- InitialPosition.Latest, true /* replicateSubscriptionState */, null);
+ topic.createSubscription(update.getSubscriptionName(), InitialPosition.Earliest,
+ true /* replicateSubscriptionState */, Collections.emptyMap())
+ .thenAccept(subscriptionCreated -> {
+ subscriptionCreated.acknowledgeMessage(Collections.singletonList(pos),
+ AckType.Cumulative, Collections.emptyMap());
+ });
}
}
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java
index 89af6f6be882f..f602659400f00 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java
@@ -28,6 +28,7 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.broker.service.BrokerService;
+import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.policies.data.Policies;
import org.apache.pulsar.common.policies.data.SubscribeRate;
import org.apache.pulsar.common.util.RateLimiter;
@@ -159,14 +160,21 @@ public void onSubscribeRateUpdate(SubscribeRate subscribeRate) {
}
/**
- * Gets configured subscribe-rate from namespace policies. Returns null if subscribe-rate is not configured
- *
- * @return
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call.
*/
+ @Deprecated
public SubscribeRate getPoliciesSubscribeRate() {
return getPoliciesSubscribeRate(brokerService, topicName);
}
+ /**
+ * @deprecated Avoid using the deprecated method
+ * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking
+ * call.
+ */
+ @Deprecated
public static SubscribeRate getPoliciesSubscribeRate(BrokerService brokerService, final String topicName) {
final String cluster = brokerService.pulsar().getConfiguration().getClusterName();
final Optional policies = DispatchRateLimiter.getPolicies(brokerService, topicName);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
index 2521bc9e754ac..c096d3610667e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java
@@ -42,10 +42,20 @@ public class RawBatchConverter {
public static boolean isReadableBatch(RawMessage msg) {
ByteBuf payload = msg.getHeadersAndPayload();
MessageMetadata metadata = Commands.parseMessageMetadata(payload);
+ return isReadableBatch(metadata);
+ }
+
+ public static boolean isReadableBatch(MessageMetadata metadata) {
return metadata.hasNumMessagesInBatch() && metadata.getEncryptionKeysCount() == 0;
}
public static List> extractIdsAndKeysAndSize(RawMessage msg)
+ throws IOException {
+ return extractIdsAndKeysAndSize(msg, true);
+ }
+
+ public static List> extractIdsAndKeysAndSize(RawMessage msg,
+ boolean extractNullKey)
throws IOException {
checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
@@ -69,7 +79,7 @@ public static List> extractIdsAndKey
msg.getMessageIdData().getEntryId(),
msg.getMessageIdData().getPartition(),
i);
- if (!smm.isCompactedOut()) {
+ if (!smm.isCompactedOut() && (extractNullKey || smm.hasPartitionKey())) {
idsAndKeysAndSize.add(ImmutableTriple.of(id,
smm.hasPartitionKey() ? smm.getPartitionKey() : null,
smm.hasPayloadSize() ? smm.getPayloadSize() : 0));
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
index 2431d14209816..7188ee6aa88bf 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java
@@ -119,7 +119,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor,
return CompletableFuture.completedFuture(null);
} else {
long endPoint = Math.min(context.ledger.getLastAddConfirmed(),
- startPoint + numberOfEntriesToRead);
+ startPoint + (numberOfEntriesToRead - 1));
if (startPoint == NEWER_THAN_COMPACTED) {
cursor.seek(compactionHorizon.getNext());
callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx);
diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
index ded798c49d5dd..5d9ae3cf9a163 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
@@ -127,22 +127,28 @@ private void phaseOneLoop(RawReader reader,
boolean deletedMessage = false;
boolean replaceMessage = false;
mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes());
- if (RawBatchConverter.isReadableBatch(m)) {
+ MessageMetadata metadata = Commands.parseMessageMetadata(m.getHeadersAndPayload());
+ if (RawBatchConverter.isReadableBatch(metadata)) {
try {
+ int numMessagesInBatch = metadata.getNumMessagesInBatch();
+ int deleteCnt = 0;
for (ImmutableTriple e : RawBatchConverter
- .extractIdsAndKeysAndSize(m)) {
+ .extractIdsAndKeysAndSize(m, false)) {
if (e != null) {
if (e.getRight() > 0) {
MessageId old = latestForKey.put(e.getMiddle(), e.getLeft());
- replaceMessage = old != null;
+ if (old != null) {
+ mxBean.addCompactionRemovedEvent(reader.getTopic());
+ }
} else {
- deletedMessage = true;
latestForKey.remove(e.getMiddle());
+ deleteCnt++;
+ mxBean.addCompactionRemovedEvent(reader.getTopic());
}
}
- if (replaceMessage || deletedMessage) {
- mxBean.addCompactionRemovedEvent(reader.getTopic());
- }
+ }
+ if (deleteCnt == numMessagesInBatch) {
+ deletedMessage = true;
}
} catch (IOException ioe) {
log.info("Error decoding batch for message {}. Whole batch will be included in output",
@@ -273,11 +279,14 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map
}
});
if (to.equals(id)) {
+ // make sure all inflight writes have finished
+ outstanding.acquire(MAX_OUTSTANDING);
addFuture.whenComplete((res, exception2) -> {
if (exception2 == null) {
promise.complete(null);
}
});
+ return;
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
index 36f27877360d8..f3e3188de4bac 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java
@@ -2953,6 +2953,7 @@ public void testLoopCreateAndDeleteTopicPolicies() throws Exception {
});
}
}
+
@Test
public void testGlobalTopicPolicies() throws Exception {
final String topic = testTopic + UUID.randomUUID();
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
similarity index 91%
rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java
rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
index 6b5898f4174e4..dae160b1421f0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java
@@ -16,9 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.broker.loadbalance;
+package org.apache.pulsar.broker.loadbalance.impl;
+import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -51,12 +53,12 @@
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils;
+import org.apache.pulsar.broker.loadbalance.LoadData;
+import org.apache.pulsar.broker.loadbalance.LoadManager;
import org.apache.pulsar.broker.TimeAverageMessageData;
import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate;
-import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
-import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper;
-import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies;
import org.apache.pulsar.client.admin.Namespaces;
import org.apache.pulsar.client.admin.PulsarAdmin;
import org.apache.pulsar.common.naming.NamespaceBundle;
@@ -329,38 +331,70 @@ public void testLoadShedding() throws Exception {
bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]);
return null;
}).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+
+ AtomicReference> selectedBrokerRef = new AtomicReference<>();
+ ModularLoadManagerImpl primaryLoadManagerSpy = spy(primaryLoadManager);
+ doAnswer(invocation -> {
+ ServiceUnitId serviceUnitId = (ServiceUnitId) invocation.getArguments()[0];
+ Optional broker = primaryLoadManager.selectBroker(serviceUnitId);
+ selectedBrokerRef.set(broker);
+ return broker;
+ }).when(primaryLoadManagerSpy).selectBroker(any());
+
setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1);
pulsar1.getConfiguration().setLoadBalancerEnabled(true);
- final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData");
+ final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData");
final Map brokerDataMap = loadData.getBrokerData();
final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost));
when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData);
brokerDataMap.put(primaryHost, brokerDataSpy1);
// Need to update all the bundle data for the shredder to see the spy.
- primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080"));
+ primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080"));
Thread.sleep(100);
localBrokerData.setCpu(new ResourceUsage(80, 100));
- primaryLoadManager.doLoadShedding();
+ primaryLoadManagerSpy.doLoadShedding();
// 80% is below overload threshold: verify nothing is unloaded.
- verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+ verify(namespacesSpy1, Mockito.times(0))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
localBrokerData.setCpu(new ResourceUsage(90, 100));
- primaryLoadManager.doLoadShedding();
+ primaryLoadManagerSpy.doLoadShedding();
// Most expensive bundle will be unloaded.
- verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+ verify(namespacesSpy1, Mockito.times(1))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
assertEquals(bundleReference.get(), mockBundleName(2));
+ assertEquals(selectedBrokerRef.get().get(), secondaryHost);
- primaryLoadManager.doLoadShedding();
+ primaryLoadManagerSpy.doLoadShedding();
// Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be
// unloaded, but this is not the case due to the spy's behavior).
- verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+ verify(namespacesSpy1, Mockito.times(2))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
assertEquals(bundleReference.get(), mockBundleName(1));
+ assertEquals(selectedBrokerRef.get().get(), secondaryHost);
- primaryLoadManager.doLoadShedding();
+ primaryLoadManagerSpy.doLoadShedding();
// Now both are in grace period: neither should be unloaded.
- verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+ verify(namespacesSpy1, Mockito.times(2))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+ assertEquals(selectedBrokerRef.get().get(), secondaryHost);
+
+ // Test bundle transfer to same broker
+
+ loadData.getRecentlyUnloadedBundles().clear();
+ primaryLoadManagerSpy.doLoadShedding();
+ verify(namespacesSpy1, Mockito.times(3))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+
+ doReturn(Optional.of(primaryHost)).when(primaryLoadManagerSpy).selectBroker(any());
+ loadData.getRecentlyUnloadedBundles().clear();
+ primaryLoadManagerSpy.doLoadShedding();
+ // The bundle shouldn't be unloaded because the broker is the same.
+ verify(namespacesSpy1, Mockito.times(3))
+ .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString());
+
}
// Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java
index 5ae6076274826..5906c6f4fe216 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java
@@ -142,6 +142,9 @@ public void testLookupTopicNotExist() throws Exception {
doReturn(uri).when(uriInfo).getRequestUri();
doReturn(true).when(config).isAuthorizationEnabled();
+ BrokerService brokerService = pulsar.getBrokerService();
+ doReturn(CompletableFuture.completedFuture(false))
+ .when(brokerService).isAllowAutoTopicCreationAsync(any());
NamespaceService namespaceService = pulsar.getNamespaceService();
CompletableFuture future = new CompletableFuture<>();
future.complete(false);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java
new file mode 100644
index 0000000000000..3110120c6946e
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.client.api.Producer;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker")
+public class BkEnsemblesChaosTest extends CanReconnectZKClientPulsarServiceBaseTest {
+
+ @Override
+ @BeforeClass(alwaysRun = true, timeOut = 300000)
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @Override
+ @AfterClass(alwaysRun = true, timeOut = 300000)
+ public void cleanup() throws Exception {
+ super.cleanup();
+ }
+
+ @Test
+ public void testBookieInfoIsCorrectEvenIfLostNotificationDueToZKClientReconnect() throws Exception {
+ final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_");
+ final byte[] msgValue = "test".getBytes();
+ admin.topics().createNonPartitionedTopic(topicName);
+ // Ensure broker works.
+ Producer producer1 = client.newProducer().topic(topicName).create();
+ producer1.send(msgValue);
+ producer1.close();
+ admin.topics().unload(topicName);
+
+ // Restart some bookies, which triggers the ZK node of Bookie deleted and created.
+ // And make the local metadata store reconnect to lose some notification of the ZK node change.
+ for (int i = 0; i < numberOfBookies - 1; i++){
+ bkEnsemble.stopBK(i);
+ }
+ makeLocalMetadataStoreKeepReconnect();
+ for (int i = 0; i < numberOfBookies - 1; i++){
+ bkEnsemble.startBK(i);
+ }
+ // Sleep 100ms to lose the notifications of ZK node create.
+ Thread.sleep(100);
+ stopLocalMetadataStoreAlwaysReconnect();
+
+ // Ensure broker still works.
+ admin.topics().unload(topicName);
+ Producer producer2 = client.newProducer().topic(topicName).create();
+ producer2.send(msgValue);
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
new file mode 100644
index 0000000000000..614b4f0837022
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import static org.testng.Assert.assertEquals;
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
+import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
+import org.apache.pulsar.common.policies.data.TopicType;
+import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.awaitility.reflect.WhiteboxImpl;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker")
+public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBaseTest {
+
+ @Override
+ @BeforeClass(alwaysRun = true, timeOut = 300000)
+ public void setup() throws Exception {
+ super.setup();
+ }
+
+ @Override
+ @AfterClass(alwaysRun = true, timeOut = 300000)
+ public void cleanup() throws Exception {
+ super.cleanup();
+ }
+
+ @Test
+ public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception {
+ final String configMetadataStoreConnectString =
+ WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString");
+ final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null);
+ // Set policy of auto create topic to PARTITIONED.
+ final String ns = defaultTenant + "/ns_" + UUID.randomUUID().toString().replaceAll("-", "");
+ final TopicName topicName1 = TopicName.get("persistent://" + ns + "/tp1");
+ final TopicName topicName2 = TopicName.get("persistent://" + ns + "/tp2");
+ admin.namespaces().createNamespace(ns);
+ AutoTopicCreationOverride autoTopicCreationOverride =
+ new AutoTopicCreationOverrideImpl.AutoTopicCreationOverrideImplBuilder().allowAutoTopicCreation(true)
+ .topicType(TopicType.PARTITIONED.toString())
+ .defaultNumPartitions(3).build();
+ admin.namespaces().setAutoTopicCreationAsync(ns, autoTopicCreationOverride);
+ // Make the cache of namespace policy is valid.
+ admin.namespaces().getAutoSubscriptionCreation(ns);
+ // Trigger the zk node "/admin/partitioned-topics/{namespace}/persistent" created.
+ admin.topics().createPartitionedTopic(topicName1.toString(), 2);
+ admin.topics().deletePartitionedTopic(topicName1.toString());
+
+ // Since there is no partitioned metadata created, the partitions count of metadata will be 0.
+ PartitionedTopicMetadata partitionedTopicMetadata1 =
+ pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get();
+ assertEquals(partitionedTopicMetadata1.partitions, 0);
+
+ // Create the partitioned metadata by another zk client.
+ // Make a error to make the cache could not update.
+ makeLocalMetadataStoreKeepReconnect();
+ anotherZKCli.create("/admin/partitioned-topics/" + ns + "/persistent/" + topicName2.getLocalName(),
+ "{\"partitions\":3}".getBytes(StandardCharsets.UTF_8),
+ ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ stopLocalMetadataStoreAlwaysReconnect();
+
+ // Get the partitioned metadata from cache, there is 90% chance that partitions count of metadata is 0.
+ PartitionedTopicMetadata partitionedTopicMetadata2 =
+ pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get();
+ // Note: If you want to reproduce the issue, you can perform validation on the next line.
+ // assertEquals(partitionedTopicMetadata2.partitions, 0);
+
+ // Verify the new method will return a correct result.
+ PartitionedTopicMetadata partitionedTopicMetadata3 =
+ pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2, true).get();
+ assertEquals(partitionedTopicMetadata3.partitions, 3);
+
+ // cleanup.
+ admin.topics().deletePartitionedTopic(topicName2.toString());
+ anotherZKCli.close();
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
index e6c12bbc5785f..579db85dc249b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java
@@ -1157,7 +1157,7 @@ public void testConcurrentLoadTopicExceedLimitShouldNotBeAutoCreated() throws Ex
} else {
// check topic should not be created if disable autoCreateTopic.
Assert.assertTrue(load.isDone());
- Assert.assertTrue(load.get().isPresent());
+ Assert.assertFalse(load.get().isPresent());
}
}
} finally {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
new file mode 100644
index 0000000000000..49fd1eb051009
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.service;
+
+import com.google.common.collect.Sets;
+import io.netty.channel.Channel;
+import java.net.URL;
+import java.nio.channels.SelectionKey;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.client.admin.PulsarAdmin;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.common.policies.data.ClusterData;
+import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.policies.data.TopicType;
+import org.apache.pulsar.metadata.impl.ZKMetadataStore;
+import org.apache.pulsar.tests.TestRetrySupport;
+import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
+import org.apache.pulsar.zookeeper.ZookeeperServerTest;
+import org.apache.zookeeper.ClientCnxn;
+import org.apache.zookeeper.ZooKeeper;
+import org.awaitility.reflect.WhiteboxImpl;
+
+@Slf4j
+public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetrySupport {
+
+ protected final String defaultTenant = "public";
+ protected final String defaultNamespace = defaultTenant + "/default";
+ protected int numberOfBookies = 3;
+ protected final String clusterName = "r1";
+ protected URL url;
+ protected URL urlTls;
+ protected ServiceConfiguration config = new ServiceConfiguration();
+ protected ZookeeperServerTest brokerConfigZk;
+ protected LocalBookkeeperEnsemble bkEnsemble;
+ protected PulsarService pulsar;
+ protected BrokerService broker;
+ protected PulsarAdmin admin;
+ protected PulsarClient client;
+ protected ZooKeeper localZkOfBroker;
+ protected Object localMetaDataStoreClientCnx;
+ protected final AtomicBoolean LocalMetadataStoreInReconnectFinishSignal = new AtomicBoolean();
+ protected void startZKAndBK() throws Exception {
+ // Start ZK.
+ brokerConfigZk = new ZookeeperServerTest(0);
+ brokerConfigZk.start();
+
+ // Start BK.
+ bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0);
+ bkEnsemble.start();
+ }
+
+ protected void startBrokers() throws Exception {
+ // Start brokers.
+ setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk);
+ pulsar = new PulsarService(config);
+ pulsar.start();
+ broker = pulsar.getBrokerService();
+ ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore();
+ localZkOfBroker = zkMetadataStore.getZkClient();
+ ClientCnxn cnxn = WhiteboxImpl.getInternalState(localZkOfBroker, "cnxn");
+ Object sendThread = WhiteboxImpl.getInternalState(cnxn, "sendThread");
+ localMetaDataStoreClientCnx = WhiteboxImpl.getInternalState(sendThread, "clientCnxnSocket");
+
+ url = new URL(pulsar.getWebServiceAddress());
+ urlTls = new URL(pulsar.getWebServiceAddressTls());
+ admin = PulsarAdmin.builder().serviceHttpUrl(url.toString()).build();
+ client = PulsarClient.builder().serviceUrl(url.toString()).build();
+ }
+
+ protected void makeLocalMetadataStoreKeepReconnect() throws Exception {
+ if (!LocalMetadataStoreInReconnectFinishSignal.compareAndSet(false, true)) {
+ throw new RuntimeException("Local metadata store is already keeping reconnect");
+ }
+ if (localMetaDataStoreClientCnx.getClass().getSimpleName().equals("ClientCnxnSocketNIO")) {
+ makeLocalMetadataStoreKeepReconnectNIO();
+ } else {
+ // ClientCnxnSocketNetty.
+ makeLocalMetadataStoreKeepReconnectNetty();
+ }
+ }
+
+ protected void makeLocalMetadataStoreKeepReconnectNIO() {
+ new Thread(() -> {
+ while (LocalMetadataStoreInReconnectFinishSignal.get()) {
+ try {
+ SelectionKey sockKey = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "sockKey");
+ if (sockKey != null) {
+ sockKey.channel().close();
+ }
+ // Prevents high cpu usage.
+ Thread.sleep(5);
+ } catch (Exception e) {
+ log.error("Try close the ZK connection of local metadata store failed: {}", e.toString());
+ }
+ }
+ }).start();
+ }
+
+ protected void makeLocalMetadataStoreKeepReconnectNetty() {
+ new Thread(() -> {
+ while (LocalMetadataStoreInReconnectFinishSignal.get()) {
+ try {
+ Channel channel = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "channel");
+ if (channel != null) {
+ channel.close();
+ }
+ // Prevents high cpu usage.
+ Thread.sleep(5);
+ } catch (Exception e) {
+ log.error("Try close the ZK connection of local metadata store failed: {}", e.toString());
+ }
+ }
+ }).start();
+ }
+
+ protected void stopLocalMetadataStoreAlwaysReconnect() {
+ LocalMetadataStoreInReconnectFinishSignal.set(false);
+ }
+
+ protected void createDefaultTenantsAndClustersAndNamespace() throws Exception {
+ admin.clusters().createCluster(clusterName, ClusterData.builder()
+ .serviceUrl(url.toString())
+ .serviceUrlTls(urlTls.toString())
+ .brokerServiceUrl(pulsar.getBrokerServiceUrl())
+ .brokerServiceUrlTls(pulsar.getBrokerServiceUrlTls())
+ .brokerClientTlsEnabled(false)
+ .build());
+
+ admin.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(),
+ Sets.newHashSet(clusterName)));
+
+ admin.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(clusterName));
+ }
+
+ @Override
+ protected void setup() throws Exception {
+ incrementSetupNumber();
+
+ log.info("--- Starting OneWayReplicatorTestBase::setup ---");
+
+ startZKAndBK();
+
+ startBrokers();
+
+ createDefaultTenantsAndClustersAndNamespace();
+
+ Thread.sleep(100);
+ log.info("--- OneWayReplicatorTestBase::setup completed ---");
+ }
+
+ private void setConfigDefaults(ServiceConfiguration config, String clusterName,
+ LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) {
+ config.setClusterName(clusterName);
+ config.setAdvertisedAddress("localhost");
+ config.setWebServicePort(Optional.of(0));
+ config.setWebServicePortTls(Optional.of(0));
+ config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort());
+ config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo");
+ config.setBrokerDeleteInactiveTopicsEnabled(false);
+ config.setBrokerDeleteInactiveTopicsFrequencySeconds(60);
+ config.setBrokerShutdownTimeoutMs(0L);
+ config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d));
+ config.setBrokerServicePort(Optional.of(0));
+ config.setBrokerServicePortTls(Optional.of(0));
+ config.setBacklogQuotaCheckIntervalInSeconds(5);
+ config.setDefaultNumberOfNamespaceBundles(1);
+ config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED.toString());
+ config.setEnableReplicatedSubscriptions(true);
+ config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000);
+ }
+
+ @Override
+ protected void cleanup() throws Exception {
+ markCurrentSetupNumberCleaned();
+ log.info("--- Shutting down ---");
+
+ stopLocalMetadataStoreAlwaysReconnect();
+
+ // Stop brokers.
+ client.close();
+ admin.close();
+ if (pulsar != null) {
+ pulsar.close();
+ }
+
+ // Stop ZK and BK.
+ bkEnsemble.stop();
+ brokerConfigZk.stop();
+
+ // Reset configs.
+ config = new ServiceConfiguration();
+ setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk);
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
index 83f4040aa40e7..73a8aca13a9a0 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java
@@ -19,13 +19,19 @@
package org.apache.pulsar.broker.service;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.common.policies.data.TopicStats;
import org.junit.Assert;
+import org.awaitility.Awaitility;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -75,4 +81,29 @@ public void testReplicatorProducerStatInTopic() throws Exception {
admin2.topics().delete(topicName);
});
}
+
+ @Test
+ public void testCreateRemoteConsumerFirst() throws Exception {
+ final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_");
+ Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create();
+ // Wait for replicator started.
+ Awaitility.await().untilAsserted(() -> {
+ Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get();
+ assertTrue(topicOptional2.isPresent());
+ PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get();
+ assertFalse(persistentTopic2.getProducers().isEmpty());
+ });
+ // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。
+ // Verify: the consumer of this cluster2 can create successfully.
+ Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1")
+ .subscribe();;
+
+ // cleanup.
+ producer1.close();
+ consumer2.close();
+ cleanupTopics(() -> {
+ admin1.topics().delete(topicName);
+ admin2.topics().delete(topicName);
+ });
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java
index 046adaa5ec257..250d971b9fb95 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java
@@ -24,10 +24,12 @@
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
import com.google.common.collect.Sets;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Map;
@@ -41,6 +43,7 @@
import org.apache.pulsar.broker.service.persistent.ReplicatedSubscriptionsController;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
@@ -48,6 +51,7 @@
import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
import org.apache.pulsar.common.policies.data.TopicStats;
+import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap;
import org.awaitility.Awaitility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -154,6 +158,93 @@ public void testReplicatedSubscriptionAcrossTwoRegions() throws Exception {
"messages don't match.");
}
+ @Test
+ public void testReplicatedSubscribeAndSwitchToStandbyCluster() throws Exception {
+ final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns_");
+ final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_");
+ final String subscriptionName = "s1";
+ final boolean isReplicatedSubscription = true;
+ final int messagesCount = 20;
+ final LinkedHashSet sentMessages = new LinkedHashSet<>();
+ final Set receivedMessages = Collections.synchronizedSet(new LinkedHashSet<>());
+ admin1.namespaces().createNamespace(namespace);
+ admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
+ admin1.topics().createNonPartitionedTopic(topicName);
+ admin1.topics().createSubscription(topicName, subscriptionName, MessageId.earliest, isReplicatedSubscription);
+ final PersistentTopic topic1 =
+ (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get();
+
+ // Send messages
+ // Wait for the topic created on the cluster2.
+ // Wait for the snapshot created.
+ final PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
+ Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create();
+ Consumer consumer1 = client1.newConsumer(Schema.STRING).topic(topicName)
+ .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe();
+ for (int i = 0; i < messagesCount / 2; i++) {
+ String msg = i + "";
+ producer1.send(msg);
+ sentMessages.add(msg);
+ }
+ Awaitility.await().untilAsserted(() -> {
+ ConcurrentOpenHashMap replicators = topic1.getReplicators();
+ assertTrue(replicators != null && replicators.size() == 1, "Replicator should started");
+ assertTrue(replicators.values().iterator().next().isConnected(), "Replicator should be connected");
+ assertTrue(topic1.getReplicatedSubscriptionController().get().getLastCompletedSnapshotId().isPresent(),
+ "One snapshot should be finished");
+ });
+ final PersistentTopic topic2 =
+ (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get();
+ Awaitility.await().untilAsserted(() -> {
+ assertTrue(topic2.getReplicatedSubscriptionController().isPresent(),
+ "Replicated subscription controller should created");
+ });
+ for (int i = messagesCount / 2; i < messagesCount; i++) {
+ String msg = i + "";
+ producer1.send(msg);
+ sentMessages.add(msg);
+ }
+
+ // Consume half messages and wait the subscription created on the cluster2.
+ for (int i = 0; i < messagesCount / 2; i++){
+ Message message = consumer1.receive(2, TimeUnit.SECONDS);
+ if (message == null) {
+ fail("Should not receive null.");
+ }
+ receivedMessages.add(message.getValue());
+ consumer1.acknowledge(message);
+ }
+ Awaitility.await().untilAsserted(() -> {
+ assertNotNull(topic2.getSubscriptions().get(subscriptionName), "Subscription should created");
+ });
+
+ // Switch client to cluster2.
+ // Since the cluster1 was not crash, all messages will be replicated to the cluster2.
+ consumer1.close();
+ final PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).build();
+ final Consumer consumer2 = client2.newConsumer(Schema.AUTO_CONSUME()).topic(topicName)
+ .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe();
+
+ // Verify all messages will be consumed.
+ Awaitility.await().untilAsserted(() -> {
+ while (true) {
+ Message message = consumer2.receive(2, TimeUnit.SECONDS);
+ if (message != null) {
+ receivedMessages.add(message.getValue().toString());
+ consumer2.acknowledge(message);
+ } else {
+ break;
+ }
+ }
+ assertEquals(receivedMessages.size(), sentMessages.size());
+ });
+
+ consumer2.close();
+ producer1.close();
+ client1.close();
+ client2.close();
+ }
+
/**
* If there's no traffic, the snapshot creation should stop and then resume when traffic comes back
*/
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java
index e06c256ccb558..c90c4a3a5bc31 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java
@@ -33,10 +33,13 @@
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException;
import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory;
@@ -53,13 +56,16 @@
import org.apache.pulsar.common.policies.data.TenantInfoImpl;
import org.apache.pulsar.common.policies.data.TopicPolicies;
import org.apache.pulsar.common.util.FutureUtil;
+import org.assertj.core.api.Assertions;
import org.awaitility.Awaitility;
+import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test(groups = "broker")
+@Slf4j
public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServiceBaseTest {
private static final String NAMESPACE1 = "system-topic/namespace-1";
@@ -385,4 +391,71 @@ public void testHandleNamespaceBeingDeleted() throws Exception {
});
service.deleteTopicPoliciesAsync(TOPIC1).get();
}
+
+ @Test
+ public void testGetTopicPoliciesWithCleanCache() throws Exception {
+ final String topic = "persistent://" + NAMESPACE1 + "/test" + UUID.randomUUID();
+ pulsarClient.newProducer().topic(topic).create().close();
+
+ SystemTopicBasedTopicPoliciesService topicPoliciesService =
+ (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService();
+
+ ConcurrentHashMap spyPoliciesCache = spy(new ConcurrentHashMap());
+ FieldUtils.writeDeclaredField(topicPoliciesService, "policiesCache", spyPoliciesCache, true);
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertThat(topicPoliciesService.getTopicPolicies(TopicName.get(topic))).isNull();
+ });
+
+ admin.topicPolicies().setMaxConsumersPerSubscription(topic, 1);
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNotNull();
+ });
+
+ Map>> readers =
+ (Map>>)
+ FieldUtils.readDeclaredField(topicPoliciesService, "readerCaches", true);
+
+ Mockito.doAnswer(invocation -> {
+ Thread.sleep(1000);
+ return invocation.callRealMethod();
+ }).when(spyPoliciesCache).get(Mockito.any());
+
+ CompletableFuture result = new CompletableFuture<>();
+ Thread thread = new Thread(() -> {
+ TopicPolicies topicPolicies;
+ for (int i = 0; i < 10; i++) {
+ try {
+ topicPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic));
+ Assert.assertNotNull(topicPolicies);
+ Thread.sleep(500);
+ } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) {
+ log.warn("topic policies cache not init, retry...");
+ } catch (Throwable e) {
+ log.error("ops: ", e);
+ result.completeExceptionally(e);
+ return;
+ }
+ }
+ result.complete(null);
+ });
+
+ Thread thread2 = new Thread(() -> {
+ for (int i = 0; i < 10; i++) {
+ CompletableFuture> readerCompletableFuture =
+ readers.get(TopicName.get(topic).getNamespaceObject());
+ if (readerCompletableFuture != null) {
+ readerCompletableFuture.join().closeAsync().join();
+ }
+ }
+ });
+
+ thread.start();
+ thread2.start();
+
+ thread.join();
+ thread2.join();
+
+ result.join();
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java
index c7cf44cb07acc..93590bb4ff583 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java
@@ -505,6 +505,43 @@ public void testDisableNamespacePolicyTakeSnapshot() throws Exception {
}
+ @Test(timeOut = 30000)
+ public void testDisableNamespacePolicyTakeSnapshotShouldNotThrowException() throws Exception {
+ cleanup();
+ conf.setBrokerDeduplicationEnabled(true);
+ conf.setBrokerDeduplicationSnapshotFrequencyInSeconds(1);
+ conf.setBrokerDeduplicationSnapshotIntervalSeconds(1);
+ conf.setBrokerDeduplicationEntriesInterval(20000);
+ setup();
+
+ final String topicName = testTopic + UUID.randomUUID().toString();
+ final String producerName = "my-producer";
+ @Cleanup
+ Producer producer = pulsarClient
+ .newProducer(Schema.STRING).topic(topicName).enableBatching(false).producerName(producerName).create();
+
+ // disable deduplication
+ admin.namespaces().setDeduplicationStatus(myNamespace, false);
+
+ int msgNum = 50;
+ CountDownLatch countDownLatch = new CountDownLatch(msgNum);
+ for (int i = 0; i < msgNum; i++) {
+ producer.newMessage().value("msg" + i).sendAsync().whenComplete((res, e) -> countDownLatch.countDown());
+ }
+ countDownLatch.await();
+ PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService()
+ .getTopicIfExists(topicName).get().get();
+ ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor();
+
+ // when disable topic deduplication the cursor should be deleted.
+ assertNull(managedCursor);
+
+ // this method will be called at brokerService forEachTopic.
+ // if topic level disable deduplication.
+ // this method should be skipped without throw exception.
+ persistentTopic.checkDeduplicationSnapshot();
+ }
+
private void waitCacheInit(String topicName) throws Exception {
pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe().close();
TopicName topic = TopicName.get(topicName);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 16a56af203973..577b9d5a6488a 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -39,6 +39,8 @@
import com.google.common.collect.Sets;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandlerContext;
import io.netty.util.Timeout;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -79,6 +81,8 @@
import org.apache.bookkeeper.mledger.impl.cache.EntryCache;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.broker.BrokerTestUtil;
+import org.apache.pulsar.broker.service.ServerCnx;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.schema.GenericRecord;
@@ -4474,4 +4478,50 @@ public void testListenerOrdering() throws Exception {
assertEquals(values.get(i), "msg-" + i);
}
}
+
+ @Test
+ public void testConsumeWhenDeliveryFailedByIOException() throws Exception {
+ final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_");
+ final String subscriptionName = "subscription1";
+ final int messagesCount = 100;
+ final int receiverQueueSize = 1;
+ Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).enableBatching(false).create();
+ ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer(Schema.STRING).topic(topic)
+ .subscriptionName(subscriptionName).receiverQueueSize(receiverQueueSize).subscribe();
+ for (int i = 0; i < messagesCount; i++) {
+ producer.send(i + "");
+ }
+ // Wait incoming queue of the consumer is full.
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(consumer.getIncomingMessageSize(), receiverQueueSize);
+ });
+
+ // Mock an io error for sending messages out.
+ ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService().getTopic(topic, false).join().get()
+ .getSubscription(subscriptionName).getDispatcher().getConsumers().iterator().next().cnx();
+ serverCnx.ctx().channel().pipeline().addFirst(new ChannelDuplexHandler() {
+
+ @Override
+ public void flush(ChannelHandlerContext ctx) throws Exception {
+ throw new IOException("Mocked error");
+ }
+ });
+
+ // Verify all messages will be consumed.
+ Set receivedMessages = new HashSet<>();
+ while (true) {
+ Message msg = consumer.receive(2, TimeUnit.SECONDS);
+ if (msg != null) {
+ receivedMessages.add(msg.getValue());
+ consumer.acknowledge(msg);
+ } else {
+ break;
+ }
+ }
+ Assert.assertEquals(receivedMessages.size(), messagesCount);
+
+ producer.close();
+ consumer.close();
+ admin.topics().delete(topic, false);
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java
new file mode 100644
index 0000000000000..5068a8ccb6b0b
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl;
+
+import lombok.Cleanup;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import java.util.List;
+import java.util.UUID;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride;
+import org.apache.pulsar.common.policies.data.Policies;
+import org.apache.pulsar.metadata.api.MetadataCache;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "broker-impl")
+@Slf4j
+public class HierarchyTopicAutoCreationTest extends ProducerConsumerBase {
+
+ @Override
+ @BeforeMethod
+ protected void setup() throws Exception {
+ super.internalSetup();
+ super.producerBaseSetup();
+ }
+
+ @Override
+ @AfterMethod(alwaysRun = true)
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test(invocationCount = 3)
+ @SneakyThrows
+ public void testPartitionedTopicAutoCreation() {
+ // Create namespace
+ final String namespace = "public/testPartitionedTopicAutoCreation";
+ admin.namespaces().createNamespace(namespace);
+ // Set policies
+ final AutoTopicCreationOverride expectedPolicies = AutoTopicCreationOverride.builder()
+ .allowAutoTopicCreation(true)
+ .topicType("partitioned")
+ .defaultNumPartitions(1)
+ .build();
+ admin.namespaces().setAutoTopicCreation(namespace, expectedPolicies);
+ // Double-check the policies
+ final AutoTopicCreationOverride nsAutoTopicCreationOverride = admin.namespaces()
+ .getAutoTopicCreation(namespace);
+ Assert.assertEquals(nsAutoTopicCreationOverride, expectedPolicies);
+ // Background invalidate cache
+ final MetadataCache nsCache = pulsar.getPulsarResources().getNamespaceResources().getCache();
+ final Thread t1 = new Thread(() -> {
+ while (true) {
+ nsCache.invalidate("/admin/policies/" + namespace);
+ }
+ });
+ t1.start();
+
+ // trigger auto-creation
+ final String topicName = "persistent://" + namespace + "/test-" + UUID.randomUUID();
+ @Cleanup final Producer producer = pulsarClient.newProducer()
+ .topic(topicName)
+ .create();
+ final List topics = admin.topics().getList(namespace);
+ Assert.assertEquals(topics.size(), 1); // expect only one topic
+ Assert.assertEquals(topics.get(0),
+ TopicName.get(topicName).getPartition(0).toString()); // expect partitioned topic
+
+ // double-check policies
+ final AutoTopicCreationOverride actualPolicies2 = admin.namespaces().getAutoTopicCreation(namespace);
+ Assert.assertEquals(actualPolicies2, expectedPolicies);
+
+ t1.interrupt();
+ }
+}
\ No newline at end of file
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java
new file mode 100644
index 0000000000000..692963884754e
--- /dev/null
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.broker.service.Topic;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.ProducerConsumerBase;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.common.api.proto.MessageMetadata;
+import org.apache.pulsar.common.protocol.Commands;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@Slf4j
+@Test(groups = "broker-impl")
+public class MessageChunkingDeduplicationTest extends ProducerConsumerBase {
+
+ @BeforeClass
+ @Override
+ protected void setup() throws Exception {
+ this.conf.setBrokerDeduplicationEnabled(true);
+ super.internalSetup();
+ super.producerBaseSetup();
+ }
+
+ @AfterClass(alwaysRun = true)
+ @Override
+ protected void cleanup() throws Exception {
+ super.internalCleanup();
+ }
+
+ @Test
+ public void testSendChunkMessageWithSameSequenceID() throws Exception {
+ String topicName = "persistent://my-property/my-ns/testSendChunkMessageWithSameSequenceID";
+ String producerName = "test-producer";
+ @Cleanup
+ Consumer consumer = pulsarClient
+ .newConsumer(Schema.STRING)
+ .subscriptionName("test-sub")
+ .topic(topicName)
+ .subscribe();
+ @Cleanup
+ Producer producer = pulsarClient
+ .newProducer(Schema.STRING)
+ .producerName(producerName)
+ .topic(topicName)
+ .enableChunking(true)
+ .enableBatching(false)
+ .create();
+ int messageSize = 6000; // payload size in KB
+ String message = String.join("", Collections.nCopies(messageSize * 1000, "a"));
+ producer.newMessage().value(message).sequenceId(10).send();
+ Message msg = consumer.receive(10, TimeUnit.SECONDS);
+ assertNotNull(msg);
+ assertTrue(msg.getMessageId() instanceof ChunkMessageIdImpl);
+ assertEquals(msg.getValue(), message);
+ producer.newMessage().value(message).sequenceId(10).send();
+ msg = consumer.receive(3, TimeUnit.SECONDS);
+ assertNull(msg);
+ }
+
+ @Test
+ public void testDeduplicateChunksInSingleChunkMessages() throws Exception {
+ String topicName = "persistent://my-property/my-ns/testDeduplicateChunksInSingleChunkMessage";
+ String producerName = "test-producer";
+ @Cleanup
+ Consumer consumer = pulsarClient
+ .newConsumer(Schema.STRING)
+ .subscriptionName("test-sub")
+ .topic(topicName)
+ .subscribe();
+ final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService()
+ .getTopicIfExists(topicName).get().orElse(null);
+ assertNotNull(persistentTopic);
+ sendChunk(persistentTopic, producerName, 1, 0, 2);
+ sendChunk(persistentTopic, producerName, 1, 1, 2);
+ sendChunk(persistentTopic, producerName, 1, 1, 2);
+
+ Message message = consumer.receive(15, TimeUnit.SECONDS);
+ assertEquals(message.getData().length, 2);
+
+ sendChunk(persistentTopic, producerName, 2, 0, 3);
+ sendChunk(persistentTopic, producerName, 2, 1, 3);
+ sendChunk(persistentTopic, producerName, 2, 1, 3);
+ sendChunk(persistentTopic, producerName, 2, 2, 3);
+ message = consumer.receive(20, TimeUnit.SECONDS);
+ assertEquals(message.getData().length, 3);
+ }
+
+ protected static void sendChunk(final PersistentTopic persistentTopic,
+ final String producerName,
+ final long sequenceId,
+ final Integer chunkId,
+ final Integer numChunks) {
+ final MessageMetadata metadata = new MessageMetadata();
+ metadata.setProducerName(producerName);
+ metadata.setSequenceId(sequenceId);
+ metadata.setPublishTime(System.currentTimeMillis());
+ if (chunkId != null && numChunks != null) {
+ metadata.setUuid(producerName + "-" + sequenceId);
+ metadata.setChunkId(chunkId);
+ metadata.setNumChunksFromMsg(numChunks);
+ metadata.setTotalChunkMsgSize(numChunks);
+ }
+ final ByteBuf buf = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata,
+ Unpooled.wrappedBuffer("a".getBytes()));
+ persistentTopic.publishMessage(buf, new Topic.PublishContext() {
+ @Override
+ public boolean isChunked() {
+ return chunkId != null;
+ }
+
+ @Override
+ public String getProducerName() {
+ return producerName;
+ }
+
+ public long getSequenceId() {
+ return sequenceId;
+ }
+
+ @Override
+ public void completed(Exception e, long ledgerId, long entryId) {
+ String name = producerName + "-" + sequenceId;
+ if (chunkId != null) {
+ name += "-" + chunkId + "-" + numChunks;
+ }
+ if (e == null) {
+ log.info("Sent {} to ({}, {})", name, ledgerId, entryId);
+ } else {
+ log.error("Failed to send {}: {}", name, e.getMessage());
+ }
+ }
+ });
+ }
+}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java
index c223208a8ef34..8016928b71201 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java
@@ -53,6 +53,7 @@
import org.apache.pulsar.client.api.ProducerConsumerBase;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Reader;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SizeUnit;
import org.apache.pulsar.client.impl.MessageImpl.SchemaState;
import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg;
@@ -332,6 +333,78 @@ public void testMaxPendingChunkMessages() throws Exception {
}
+ @Test
+ public void testResendChunkMessagesWithoutAckHole() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ final String topicName = "persistent://my-property/my-ns/testResendChunkMessagesWithoutAckHole";
+ final String subName = "my-subscriber-name";
+ @Cleanup
+ Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName(subName)
+ .maxPendingChunkedMessage(10)
+ .autoAckOldestChunkedMessageOnQueueFull(true)
+ .subscribe();
+ @Cleanup
+ Producer producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableChunking(true)
+ .enableBatching(false)
+ .create();
+
+ sendSingleChunk(producer, "0", 0, 2);
+
+ sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk
+ sendSingleChunk(producer, "0", 1, 2);
+
+ Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS);
+ assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|");
+ consumer.acknowledge(receivedMsg);
+ assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName)
+ .getNonContiguousDeletedMessagesRanges(), 0);
+ }
+
+ @Test
+ public void testResendChunkMessages() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+ final String topicName = "persistent://my-property/my-ns/testResendChunkMessages";
+
+ @Cleanup
+ Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+ .topic(topicName)
+ .subscriptionName("my-subscriber-name")
+ .maxPendingChunkedMessage(10)
+ .autoAckOldestChunkedMessageOnQueueFull(true)
+ .subscribe();
+ @Cleanup
+ Producer producer = pulsarClient.newProducer(Schema.STRING)
+ .topic(topicName)
+ .enableChunking(true)
+ .enableBatching(false)
+ .create();
+
+ sendSingleChunk(producer, "0", 0, 2);
+
+ sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk
+ sendSingleChunk(producer, "1", 0, 3); // This is for testing the interwoven chunked message
+ sendSingleChunk(producer, "1", 1, 3);
+ sendSingleChunk(producer, "1", 0, 3); // Resending the UUID-1 chunked message
+
+ sendSingleChunk(producer, "0", 1, 2);
+
+ Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS);
+ assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|");
+ consumer.acknowledge(receivedMsg);
+
+ sendSingleChunk(producer, "1", 1, 3);
+ sendSingleChunk(producer, "1", 2, 3);
+
+ receivedMsg = consumer.receive(5, TimeUnit.SECONDS);
+ assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|chunk-1-2|");
+ consumer.acknowledge(receivedMsg);
+ Assert.assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 8);
+ }
+
/**
* Validate that chunking is not supported with batching and non-persistent topic
*
@@ -514,4 +587,17 @@ private String createMessagePayload(int size) {
return str.toString();
}
+ private void sendSingleChunk(Producer producer, String uuid, int chunkId, int totalChunks)
+ throws PulsarClientException {
+ TypedMessageBuilderImpl msg = (TypedMessageBuilderImpl) producer.newMessage()
+ .value(String.format("chunk-%s-%d|", uuid, chunkId));
+ MessageMetadata msgMetadata = msg.getMetadataBuilder();
+ msgMetadata.setUuid(uuid)
+ .setChunkId(chunkId)
+ .setNumChunksFromMsg(totalChunks)
+ .setTotalChunkMsgSize(100);
+ msg.send();
+ }
+
+
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
index ddad53fbc829f..0eec3e31cca1d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java
@@ -22,12 +22,13 @@
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.assertFalse;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import io.netty.buffer.ByteBuf;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -45,9 +46,9 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import io.netty.buffer.ByteBuf;
import lombok.Cleanup;
import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.api.OpenBuilder;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -61,6 +62,7 @@
import org.apache.pulsar.broker.service.Topic;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.broker.service.persistent.SystemTopic;
+import org.apache.pulsar.client.admin.LongRunningProcessStatus;
import org.apache.pulsar.client.api.CompressionType;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.CryptoKeyReader;
@@ -70,11 +72,14 @@
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.ProducerBuilder;
+import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.Reader;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SubscriptionInitialPosition;
import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.BatchMessageIdImpl;
+import org.apache.pulsar.client.impl.ConsumerImpl;
import org.apache.pulsar.common.naming.NamespaceName;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.ClusterData;
@@ -91,6 +96,7 @@
import org.testng.annotations.Test;
@Test(groups = "flaky")
+@Slf4j
public class CompactionTest extends MockedPulsarServiceBaseTest {
private ScheduledExecutorService compactionScheduler;
private BookKeeper bk;
@@ -530,6 +536,61 @@ public void testBatchMessageIdsDontChange() throws Exception {
}
}
+ @Test
+ public void testBatchMessageWithNullValue() throws Exception {
+ String topic = "persistent://my-property/use/my-ns/my-topic1";
+
+ pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
+ .receiverQueueSize(1).readCompacted(true).subscribe().close();
+
+ try (Producer producer = pulsarClient.newProducer().topic(topic)
+ .maxPendingMessages(3)
+ .enableBatching(true)
+ .batchingMaxMessages(3)
+ .batchingMaxPublishDelay(1, TimeUnit.HOURS)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create()
+ ) {
+ // batch 1
+ producer.newMessage().key("key1").value("my-message-1".getBytes()).sendAsync();
+ producer.newMessage().key("key1").value(null).sendAsync();
+ producer.newMessage().key("key2").value("my-message-3".getBytes()).send();
+
+ // batch 2
+ producer.newMessage().key("key3").value("my-message-4".getBytes()).sendAsync();
+ producer.newMessage().key("key3").value("my-message-5".getBytes()).sendAsync();
+ producer.newMessage().key("key3").value("my-message-6".getBytes()).send();
+
+ // batch 3
+ producer.newMessage().key("key4").value("my-message-7".getBytes()).sendAsync();
+ producer.newMessage().key("key4").value(null).sendAsync();
+ producer.newMessage().key("key5").value("my-message-9".getBytes()).send();
+ }
+
+
+ // compact the topic
+ Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
+ compactor.compact(topic);
+
+ // Read messages before compaction to get ids
+ List> messages = new ArrayList<>();
+ try (Consumer consumer = pulsarClient.newConsumer().topic(topic)
+ .subscriptionName("sub1").receiverQueueSize(1).readCompacted(true).subscribe()) {
+ while (true) {
+ Message message = consumer.receive(5, TimeUnit.SECONDS);
+ if (message == null) {
+ break;
+ }
+ messages.add(message);
+ }
+ }
+
+ assertEquals(messages.size(), 3);
+ assertEquals(messages.get(0).getKey(), "key2");
+ assertEquals(messages.get(1).getKey(), "key3");
+ assertEquals(messages.get(2).getKey(), "key5");
+ }
+
@Test
public void testWholeBatchCompactedOut() throws Exception {
String topic = "persistent://my-property/use/my-ns/my-topic1";
@@ -1748,4 +1809,38 @@ public void addFailed(ManagedLedgerException exception, Object ctx) {
Assert.assertNotEquals(ledgerId, -1L);
});
}
+
+ @Test(timeOut = 100000)
+ public void testReceiverQueueSize() throws Exception {
+ final String topicName = "persistent://my-property/use/my-ns/testReceiverQueueSize" + UUID.randomUUID();
+ final String subName = "my-sub";
+ final int receiveQueueSize = 1;
+ @Cleanup
+ PulsarClient client = newPulsarClient(lookupUrl.toString(), 100);
+ Producer producer = pulsarClient.newProducer(Schema.STRING)
+ .enableBatching(false).topic(topicName).create();
+
+ for (int i = 0; i < 10; i++) {
+ producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync();
+ }
+ producer.flush();
+
+ admin.topics().triggerCompaction(topicName);
+
+ Awaitility.await().untilAsserted(() -> {
+ assertEquals(admin.topics().compactionStatus(topicName).status,
+ LongRunningProcessStatus.Status.SUCCESS);
+ });
+
+ ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING)
+ .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName)
+ .subscribe();
+
+ //Give some time to consume
+ Awaitility.await()
+ .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(),
+ receiveQueueSize));
+ consumer.close();
+ producer.close();
+ }
}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java
index bd875384aecb3..304aae3f57f7b 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java
@@ -19,13 +19,10 @@
package org.apache.pulsar.compaction;
import static org.apache.pulsar.client.impl.RawReaderTest.extractKey;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
import io.netty.buffer.ByteBuf;
-
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
@@ -33,22 +30,29 @@
import java.util.Map;
import java.util.Optional;
import java.util.Random;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
-
+import lombok.Cleanup;
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.impl.PositionImpl;
import org.apache.pulsar.broker.ServiceConfiguration;
import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
+import org.apache.pulsar.broker.service.persistent.PersistentTopic;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
+import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.api.RawMessage;
import org.apache.pulsar.client.impl.PulsarClientImpl;
import org.apache.pulsar.client.impl.RawMessageImpl;
import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.protocol.Commands;
import org.apache.pulsar.common.policies.data.TenantInfoImpl;
+import org.apache.pulsar.common.protocol.Commands;
+import org.awaitility.Awaitility;
import org.mockito.Mockito;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
@@ -59,7 +63,6 @@
public class CompactorTest extends MockedPulsarServiceBaseTest {
private ScheduledExecutorService compactionScheduler;
-
@BeforeMethod
@Override
public void setup() throws Exception {
@@ -82,18 +85,19 @@ public void cleanup() throws Exception {
compactionScheduler.shutdownNow();
}
- private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception {
+ private List compactAndVerify(String topic, Map expected, boolean checkMetrics)
+ throws Exception {
BookKeeper bk = pulsar.getBookKeeperClientFactory().create(
this.conf, null, null, Optional.empty(), null);
Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
long compactedLedgerId = compactor.compact(topic).get();
LedgerHandle ledger = bk.openLedger(compactedLedgerId,
- Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
- Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
+ Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE,
+ Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac
- expected.size(),
- "Should have as many entries as there is keys");
+ expected.size(),
+ "Should have as many entries as there is keys");
List keys = new ArrayList<>();
Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed());
@@ -107,7 +111,7 @@ private List compactAndVerify(String topic, Map expected
byte[] bytes = new byte[payload.readableBytes()];
payload.readBytes(bytes);
Assert.assertEquals(bytes, expected.remove(key),
- "Compacted version should match expected version");
+ "Compacted version should match expected version");
m.close();
}
if (checkMetrics) {
@@ -131,17 +135,18 @@ public void testCompaction() throws Exception {
final int numMessages = 1000;
final int maxKeys = 10;
+ @Cleanup
Producer producer = pulsarClient.newProducer().topic(topic)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
Map expected = new HashMap<>();
Random r = new Random(0);
for (int j = 0; j < numMessages; j++) {
int keyIndex = r.nextInt(maxKeys);
- String key = "key"+keyIndex;
+ String key = "key" + keyIndex;
byte[] data = ("my-message-" + key + "-" + j).getBytes();
producer.newMessage()
.key(key)
@@ -156,10 +161,11 @@ public void testCompaction() throws Exception {
public void testCompactAddCompact() throws Exception {
String topic = "persistent://my-property/use/my-ns/my-topic1";
+ @Cleanup
Producer producer = pulsarClient.newProducer().topic(topic)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
Map expected = new HashMap<>();
@@ -193,10 +199,11 @@ public void testCompactAddCompact() throws Exception {
public void testCompactedInOrder() throws Exception {
String topic = "persistent://my-property/use/my-ns/my-topic1";
+ @Cleanup
Producer producer = pulsarClient.newProducer().topic(topic)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
producer.newMessage()
.key("c")
@@ -243,6 +250,51 @@ public void testPhaseOneLoopTimeConfiguration() {
}
+ @Test
+ public void testCompactedWithConcurrentSend() throws Exception {
+ String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend";
+
+ @Cleanup
+ Producer producer = pulsarClient.newProducer().topic(topic)
+ .enableBatching(false)
+ .messageRoutingMode(MessageRoutingMode.SinglePartition)
+ .create();
+
+
+ BookKeeper bk = pulsar.getBookKeeperClientFactory().create(
+ this.conf, null, null, Optional.empty(), null);
+ Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
+
+ CompletableFuture future = CompletableFuture.runAsync(() -> {
+ for (int i = 0; i < 100; i++) {
+ try {
+ producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send();
+ } catch (PulsarClientException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get();
+ CompactedTopic compactedTopic = persistentTopic.getCompactedTopic();
+
+ Awaitility.await().untilAsserted(() -> {
+ long compactedLedgerId = compactor.compact(topic).get();
+ Thread.sleep(300);
+ Optional compactedTopicContext = persistentTopic.getCompactedTopicContext();
+ Assert.assertTrue(compactedTopicContext.isPresent());
+ Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId);
+ });
+
+ Position lastCompactedPosition = compactedTopic.getCompactionHorizon().get();
+ Entry lastCompactedEntry = compactedTopic.readLastEntryOfCompactedLedger().get();
+
+ Assert.assertTrue(PositionImpl.get(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId())
+ .compareTo(PositionImpl.get(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId())) >= 0);
+
+ future.join();
+ }
+
public ByteBuf extractPayload(RawMessage m) throws Exception {
ByteBuf payloadAndMetadata = m.getHeadersAndPayload();
Commands.skipChecksumIfPresent(payloadAndMetadata);
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
index 7133684721c9b..dcd6671780a6d 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java
@@ -1179,6 +1179,33 @@ private void testIncompatibleSchema() throws Exception {
}
}
+ /**
+ * This test just ensure that schema check still keeps the original logic: if there has any producer, but no schema
+ * was registered, the new consumer could not register new schema.
+ * TODO: I think this design should be improved: if a producer used "AUTO_PRODUCE_BYTES" schema, we should allow
+ * the new consumer to register new schema. But before we can solve this problem, we need to modify
+ * "CmdProducer" to let the Broker know that the Producer uses a schema of type "AUTO_PRODUCE_BYTES".
+ */
+ @Test
+ public void testAutoProduceAndSpecifiedConsumer() throws Exception {
+ final String namespace = PUBLIC_TENANT + "/ns_" + randomName(16);
+ admin.namespaces().createNamespace(namespace, Sets.newHashSet(CLUSTER_NAME));
+ final String topicName = "persistent://" + namespace + "/tp_" + randomName(16);
+ admin.topics().createNonPartitionedTopic(topicName);
+
+ Producer producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName).create();
+ try {
+ pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub1").subscribe();
+ fail("Should throw ex: Topic does not have schema to check");
+ } catch (Exception ex){
+ assertTrue(ex.getMessage().contains("Topic does not have schema to check"));
+ }
+
+ // Cleanup.
+ producer.close();
+ admin.topics().delete(topicName);
+ }
+
@Test
public void testCreateSchemaInParallel() throws Exception {
final String namespace = "test-namespace-" + randomName(16);
diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml
index 852d4af907490..bd90997f2b18f 100644
--- a/pulsar-client-1x-base/pom.xml
+++ b/pulsar-client-1x-base/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-1x-base
Pulsar Client 1.x Compatibility Base
pom
-
pulsar-client-2x-shaded
pulsar-client-1x
-
@@ -54,7 +50,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
@@ -70,5 +65,4 @@
-
diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml
index 311dc3445d987..8839ace7e1835 100644
--- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml
+++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-client-1x-base
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-1x
Pulsar Client 1.x Compatibility API
-
${project.groupId}
pulsar-client-2x-shaded
${project.version}
-
com.google.guava
guava
-
org.apache.commons
commons-lang3
-
-
@@ -70,7 +63,6 @@
-
com.github.spotbugs
spotbugs-maven-plugin
@@ -87,5 +79,4 @@
-
diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml
index 8b0f3f517bc7a..02011e78f5aaa 100644
--- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml
+++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-client-1x-base
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-2x-shaded
Pulsar Client 2.x Shaded API
-
${project.groupId}
@@ -39,10 +36,9 @@
${project.version}
-
-
+
org.apache.maven.plugins
maven-shade-plugin
@@ -56,16 +52,15 @@
true
true
false
-
- org.apache.pulsar:pulsar-client
- org.apache.pulsar:pulsar-client-api
+ io.streamnative:pulsar-client
+ io.streamnative:pulsar-client-api
- org.apache.pulsar:pulsar-client
+ io.streamnative:pulsar-client
**
@@ -93,6 +88,30 @@
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml
index e06d71e039c34..938db21e9002c 100644
--- a/pulsar-client-admin-api/pom.xml
+++ b/pulsar-client-admin-api/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar
- 2.10.5-SNAPSHOT
- ..
-
-
- pulsar-client-admin-api
- Pulsar Client Admin :: API
-
-
-
- ${project.groupId}
- pulsar-client-api
- ${project.version}
-
-
-
- org.slf4j
- slf4j-api
-
-
-
-
-
-
- org.gaul
- modernizer-maven-plugin
-
- true
- 8
-
-
-
- modernizer
- verify
-
- modernizer
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- checkstyle
- verify
-
- check
-
-
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar
+ 2.10.5.3
+ ..
+
+ pulsar-client-admin-api
+ Pulsar Client Admin :: API
+
+
+ ${project.groupId}
+ pulsar-client-api
+ ${project.version}
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+
+ org.gaul
+ modernizer-maven-plugin
+
+ true
+ 8
+
+
+
+ modernizer
+ verify
+
+ modernizer
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ checkstyle
+ verify
+
+ check
+
+
+
+
+
+
diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml
index 1e54993be9118..da22cdd3634dc 100644
--- a/pulsar-client-admin-shaded/pom.xml
+++ b/pulsar-client-admin-shaded/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-admin
Pulsar Client Admin
-
${project.groupId}
@@ -74,7 +70,6 @@
-
maven-antrun-plugin
@@ -87,15 +82,12 @@
-
+
-
org.apache.maven.plugins
maven-shade-plugin
@@ -108,11 +100,10 @@
true
true
-
- org.apache.pulsar:pulsar-client-original
- org.apache.pulsar:pulsar-client-admin-original
+ io.streamnative:pulsar-client-original
+ io.streamnative:pulsar-client-admin-original
org.apache.commons:commons-lang3
commons-codec:commons-codec
commons-collections:commons-collections
@@ -125,7 +116,7 @@
com.google.code.gson:gson
com.fasterxml.jackson.core
io.netty:*
- org.apache.pulsar:pulsar-common
+ io.streamnative:pulsar-common
org.apache.bookkeeper:*
com.yahoo.datasketches:sketches-core
org.glassfish.jersey*:*
@@ -147,12 +138,12 @@
org.yaml:snakeyaml
io.swagger:*
- org.apache.pulsar:pulsar-client-messagecrypto-bc
+ io.streamnative:pulsar-client-messagecrypto-bc
- org.apache.pulsar:pulsar-client-original
+ io.streamnative:pulsar-client-original
**
@@ -162,7 +153,7 @@
- org.apache.pulsar:pulsar-client-admin-original
+ io.streamnative:pulsar-client-admin-original
**
@@ -173,7 +164,7 @@
-
+
org.asynchttpclient
org.apache.pulsar.shade.org.asynchttpclient
@@ -248,51 +239,75 @@
org.reactivestreams
org.apache.pulsar.shade.org.reactivestreams
-
- io.grpc
- org.apache.pulsar.shade.io.grpc
-
-
- okio
- org.apache.pulsar.shade.okio
-
-
- com.squareup
- org.apache.pulsar.shade.com.squareup
-
-
- io.opencensus
- org.apache.pulsar.shade.io.opencensus
-
-
- org.eclipse.jetty
- org.apache.pulsar.shade.org.eclipse.jetty
-
-
- org.objenesis
- org.apache.pulsar.shade.org.objenesis
-
-
- org.yaml
- org.apache.pulsar.shade.org.yaml
-
-
- io.swagger
- org.apache.pulsar.shade.io.swagger
-
-
- org.apache.bookkeeper
- org.apache.pulsar.shade.org.apache.bookkeeper
-
+
+ io.grpc
+ org.apache.pulsar.shade.io.grpc
+
+
+ okio
+ org.apache.pulsar.shade.okio
+
+
+ com.squareup
+ org.apache.pulsar.shade.com.squareup
+
+
+ io.opencensus
+ org.apache.pulsar.shade.io.opencensus
+
+
+ org.eclipse.jetty
+ org.apache.pulsar.shade.org.eclipse.jetty
+
+
+ org.objenesis
+ org.apache.pulsar.shade.org.objenesis
+
+
+ org.yaml
+ org.apache.pulsar.shade.org.yaml
+
+
+ io.swagger
+ org.apache.pulsar.shade.io.swagger
+
+
+ org.apache.bookkeeper
+ org.apache.pulsar.shade.org.apache.bookkeeper
+
-
-
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml
index e2fe7cc91165c..6f22b0b7a6f2c 100644
--- a/pulsar-client-admin/pom.xml
+++ b/pulsar-client-admin/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-admin-original
Pulsar Client Admin Original
-
${project.groupId}
pulsar-client-original
${project.version}
-
${project.groupId}
pulsar-client-admin-api
${project.version}
-
org.glassfish.jersey.core
jersey-client
-
org.glassfish.jersey.media
jersey-media-json-jackson
-
jakarta.activation
jakarta.activation-api
-
org.glassfish.jersey.media
jersey-media-multipart
-
com.fasterxml.jackson.jaxrs
jackson-jaxrs-json-provider
-
org.glassfish.jersey.inject
jersey-hk2
-
javax.xml.bind
jaxb-api
@@ -91,29 +79,24 @@
javax.activation
runtime
-
org.slf4j
jul-to-slf4j
-
com.google.guava
guava
-
com.google.code.gson
gson
-
${project.groupId}
pulsar-package-core
${project.version}
-
@@ -133,7 +116,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml
index 062f9629f77d2..98fa3e34904ba 100644
--- a/pulsar-client-all/pom.xml
+++ b/pulsar-client-all/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-all
Pulsar Client All
-
${project.groupId}
@@ -55,10 +52,8 @@
true
-
-
org.apache.maven.plugins
maven-dependency-plugin
@@ -94,7 +89,6 @@
-
maven-antrun-plugin
@@ -107,15 +101,12 @@
-
+
-
org.apache.maven.plugins
@@ -130,12 +121,11 @@
true
true
false
-
- org.apache.pulsar:pulsar-client-original
- org.apache.pulsar:pulsar-transaction-common
- org.apache.pulsar:pulsar-client-admin-original
+ io.streamnative:pulsar-client-original
+ io.streamnative:pulsar-transaction-common
+ io.streamnative:pulsar-client-admin-original
org.apache.commons:commons-lang3
commons-codec:commons-codec
commons-collections:commons-collections
@@ -164,8 +154,7 @@
commons-*:*
io.swagger:*
io.airlift:*
-
- org.apache.pulsar:pulsar-common
+ io.streamnative:pulsar-common
org.apache.bookkeeper:*
com.yahoo.datasketches:sketches-core
org.glassfish.jersey*:*
@@ -192,12 +181,12 @@
org.apache.commons:commons-compress
org.tukaani:xz
- org.apache.pulsar:pulsar-client-messagecrypto-bc
+ io.streamnative:pulsar-client-messagecrypto-bc
- org.apache.pulsar:pulsar-client-original
+ io.streamnative:pulsar-client-original
**
@@ -292,24 +281,24 @@
org.apache.pulsar.shade.org.glassfish
- io.grpc
- org.apache.pulsar.shade.io.grpc
+ io.grpc
+ org.apache.pulsar.shade.io.grpc
- okio
- org.apache.pulsar.shade.okio
+ okio
+ org.apache.pulsar.shade.okio
- com.squareup
- org.apache.pulsar.shade.com.squareup
+ com.squareup
+ org.apache.pulsar.shade.com.squareup
- io.opencensus
- org.apache.pulsar.shade.io.opencensus
+ io.opencensus
+ org.apache.pulsar.shade.io.opencensus
- org.eclipse.jetty
- org.apache.pulsar.shade.org.eclipse.jetty
+ org.eclipse.jetty
+ org.apache.pulsar.shade.org.eclipse.jetty
org.objenesis
@@ -366,14 +355,13 @@
-
-
+
+
-
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar
- 2.10.5-SNAPSHOT
- ..
-
-
- pulsar-client-api
- Pulsar Client :: API
-
-
-
-
-
- com.google.protobuf
- protobuf-java
- provided
-
-
-
-
-
-
-
- org.gaul
- modernizer-maven-plugin
-
- true
- 8
-
-
-
- modernizer
- verify
-
- modernizer
-
-
-
-
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
- ${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- checkstyle
- verify
-
- check
-
-
-
-
-
-
-
+
+ com.google.protobuf
+ protobuf-java
+ provided
+
+
+
+
+
+ org.gaul
+ modernizer-maven-plugin
+
+ true
+ 8
+
+
+
+ modernizer
+ verify
+
+ modernizer
+
+
+
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+ ${basedir}/src/main/resources/findbugsExclude.xml
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ checkstyle
+ verify
+
+ check
+
+
+
+
+
+
diff --git a/pulsar-client-api/src/main/resources/findbugsExclude.xml b/pulsar-client-api/src/main/resources/findbugsExclude.xml
index 9d73ac29a7bdb..353f01a7bb285 100644
--- a/pulsar-client-api/src/main/resources/findbugsExclude.xml
+++ b/pulsar-client-api/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml
index d0f032db40c13..9abeb20d76dec 100644
--- a/pulsar-client-auth-athenz/pom.xml
+++ b/pulsar-client-auth-athenz/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-auth-athenz
jar
Athenz authentication plugin for java client
-
-
${project.groupId}
pulsar-client-original
${project.parent.version}
true
-
com.yahoo.athenz
athenz-zts-java-client-core
-
com.google.guava
guava
-
org.apache.commons
commons-lang3
-
@@ -77,7 +69,6 @@
-
com.github.spotbugs
spotbugs-maven-plugin
@@ -95,7 +86,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml
+++ b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-auth-sasl
jar
SASL authentication plugin for java client
-
-
${project.groupId}
pulsar-client-original
${project.parent.version}
true
-
com.google.guava
guava
-
org.apache.commons
commons-lang3
-
org.projectlombok
lombok
-
javax.ws.rs
javax.ws.rs-api
-
org.glassfish.jersey.core
jersey-client
-
-
@@ -88,7 +77,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
diff --git a/pulsar-client-cpp/eclipse-formatter.xml b/pulsar-client-cpp/eclipse-formatter.xml
index 12c2a2171bcac..41f617cc914e9 100644
--- a/pulsar-client-cpp/eclipse-formatter.xml
+++ b/pulsar-client-cpp/eclipse-formatter.xml
@@ -1,4 +1,4 @@
-
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml
index 74f14e406e8ff..e80294fc02c9f 100644
--- a/pulsar-client-messagecrypto-bc/pom.xml
+++ b/pulsar-client-messagecrypto-bc/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-messagecrypto-bc
jar
Message crypto for End to End encryption with BouncyCastleProvider
-
${project.groupId}
@@ -40,7 +37,6 @@
${project.parent.version}
provided
-
${project.groupId}
bouncy-castle-bc
@@ -48,9 +44,7 @@
pkg
true
-
-
diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml
index e28a03e99ab3a..4dc1a4518f3b5 100644
--- a/pulsar-client-shaded/pom.xml
+++ b/pulsar-client-shaded/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client
Pulsar Client Java
-
${project.groupId}
@@ -49,11 +46,8 @@
${project.version}
-
-
-
org.apache.maven.plugins
maven-dependency-plugin
@@ -89,7 +83,6 @@
-
maven-antrun-plugin
@@ -102,15 +95,12 @@
-
+
-
org.apache.maven.plugins
@@ -125,11 +115,10 @@
true
true
false
-
- org.apache.pulsar:pulsar-client-original
- org.apache.pulsar:pulsar-transaction-common
+ io.streamnative:pulsar-client-original
+ io.streamnative:pulsar-transaction-common
org.apache.bookkeeper:*
org.apache.commons:commons-lang3
commons-codec:commons-codec
@@ -157,24 +146,22 @@
commons-*:*
io.swagger:*
io.airlift:*
-
- org.apache.pulsar:pulsar-common
+ io.streamnative:pulsar-common
com.yahoo.datasketches:sketches-core
org.objenesis:*
org.yaml:snakeyaml
-
org.apache.avro:*
com.thoughtworks.paranamer:paranamer
org.apache.commons:commons-compress
org.tukaani:xz
- org.apache.pulsar:pulsar-client-messagecrypto-bc
+ io.streamnative:pulsar-client-messagecrypto-bc
- org.apache.pulsar:pulsar-client-original
+ io.streamnative:pulsar-client-original
**
@@ -299,14 +286,13 @@
-
-
+
+
-
com.github.spotbugs
spotbugs-maven-plugin
@@ -320,7 +306,6 @@
-
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-tools-test
Pulsar Client Tools Test
Pulsar Client Tools Test
-
${project.groupId}
@@ -82,7 +80,6 @@
-
org.apache.maven.plugins
maven-deploy-plugin
diff --git a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml
+++ b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-tools
Pulsar Client Tools
Pulsar Client Tools
-
com.beust
@@ -85,7 +83,6 @@
pulsar-websocket
${project.version}
-
${project.groupId}
@@ -93,15 +90,13 @@
${project.version}
test
-
- org.apache.pulsar
- pulsar-io-batch-discovery-triggerers
- ${project.version}
- test
+ io.streamnative
+ pulsar-io-batch-discovery-triggerers
+ ${project.version}
+ test
-
@@ -121,7 +116,6 @@
-
@@ -141,7 +135,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
@@ -157,5 +150,4 @@
-
diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml
index 96a259371a648..e1876d1df68cc 100644
--- a/pulsar-client/pom.xml
+++ b/pulsar-client/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-client-original
Pulsar Client Java
-
${project.groupId}
pulsar-client-api
${project.parent.version}
-
${project.groupId}
pulsar-common
${project.parent.version}
-
${project.groupId}
pulsar-transaction-common
${project.parent.version}
-
${project.groupId}
bouncy-castle-bc
${project.parent.version}
pkg
-
${project.groupId}
@@ -65,7 +58,6 @@
${project.parent.version}
true
-
io.netty
netty-codec-http
@@ -78,55 +70,44 @@
io.netty
netty-codec-socks
-
io.netty
netty-resolver-dns
-
org.apache.commons
commons-lang3
-
org.asynchttpclient
async-http-client
-
com.typesafe.netty
netty-reactive-streams
-
org.slf4j
slf4j-api
-
commons-codec
commons-codec
-
com.yahoo.datasketches
sketches-core
-
com.google.code.gson
gson
-
-
org.apache.avro
avro
${avro.version}
-
org.apache.avro
avro-protobuf
@@ -138,36 +119,30 @@
-
joda-time
joda-time
provided
-
com.google.protobuf
protobuf-java
provided
-
com.fasterxml.jackson.module
jackson-module-jsonSchema
-
net.jcip
jcip-annotations
-
com.github.spotbugs
spotbugs-annotations
provided
true
-
${project.groupId}
@@ -175,22 +150,18 @@
${project.parent.version}
test
-
org.skyscreamer
jsonassert
${skyscreamer.version}
test
-
org.awaitility
awaitility
test
-
-
@@ -217,7 +188,6 @@
-
org.apache.maven.plugins
maven-checkstyle-plugin
@@ -231,7 +201,6 @@
-
org.xolstice.maven.plugins
protobuf-maven-plugin
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java
index 9b4d1b7d683dd..f762b5ab45038 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java
@@ -35,7 +35,6 @@ public abstract class AbstractBatchMessageContainer implements BatchMessageConta
protected CompressionType compressionType;
protected CompressionCodec compressor;
protected String topicName;
- protected String producerName;
protected ProducerImpl producer;
protected int maxNumMessagesInBatch;
@@ -98,7 +97,6 @@ public ProducerImpl.OpSendMsg createOpSendMsg() throws IOException {
public void setProducer(ProducerImpl> producer) {
this.producer = producer;
this.topicName = producer.getTopic();
- this.producerName = producer.getProducerName();
this.compressionType = CompressionCodecProvider
.convertToWireProtocol(producer.getConfiguration().getCompressionType());
this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java
index 37b522d8902fa..f657816b6dd04 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java
@@ -80,8 +80,8 @@ public BatchMessageContainerImpl(ProducerImpl> producer) {
public boolean add(MessageImpl> msg, SendCallback callback) {
if (log.isDebugEnabled()) {
- log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName,
- numMessagesInBatch);
+ log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName,
+ producer.getProducerName(), numMessagesInBatch);
}
if (++numMessagesInBatch == 1) {
@@ -193,8 +193,8 @@ public void discard(Exception ex) {
firstCallback.sendComplete(ex);
}
} catch (Throwable t) {
- log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName,
- lowestSequenceId, t);
+ log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName,
+ producer.getProducerName(), lowestSequenceId, t);
}
clear();
}
@@ -226,6 +226,14 @@ public OpSendMsg createOpSendMsg() throws IOException {
ByteBufPair cmd = producer.sendMessage(producer.producerId, messageMetadata.getSequenceId(),
messageMetadata.getHighestSequenceId(), numMessagesInBatch, messageMetadata, encryptedPayload);
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] [{}] Build batch msg seq:{}, highest-seq:{}, numMessagesInBatch: {}, uncompressedSize: {},"
+ + " payloadSize: {}", topicName, producer.getProducerName(),
+ messageMetadata.getSequenceId(), messageMetadata.getNumMessagesInBatch(),
+ messageMetadata.getHighestSequenceId(),
+ messageMetadata.getUncompressedSize(), encryptedPayload.readableBytes());
+ }
+
OpSendMsg op = OpSendMsg.create(messages, cmd, messageMetadata.getSequenceId(),
messageMetadata.getHighestSequenceId(), firstCallback);
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java
index 7614728353f89..9fdd44715f669 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java
@@ -54,8 +54,8 @@ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer {
@Override
public boolean add(MessageImpl> msg, SendCallback callback) {
if (log.isDebugEnabled()) {
- log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, producerName,
- numMessagesInBatch);
+ log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName,
+ producer.getProducerName(), numMessagesInBatch);
}
numMessagesInBatch++;
currentBatchSizeBytes += msg.getDataBuffer().readableBytes();
@@ -68,7 +68,7 @@ public boolean add(MessageImpl> msg, SendCallback callback) {
part.compressor = compressor;
part.maxBatchSize = maxBatchSize;
part.topicName = topicName;
- part.producerName = producerName;
+ part.producerName = producer.getProducerName();
batches.putIfAbsent(key, part);
if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) {
@@ -104,7 +104,7 @@ public void discard(Exception ex) {
// Need to protect ourselves from any exception being thrown in the future handler from the application
batches.forEach((k, v) -> v.firstCallback.sendComplete(ex));
} catch (Throwable t) {
- log.warn("[{}] [{}] Got exception while completing the callback", topicName, producerName, t);
+ log.warn("[{}] [{}] Got exception while completing the callback", topicName, producer.getProducerName(), t);
}
batches.forEach((k, v) -> ReferenceCountUtil.safeRelease(v.batchedMessageMetadataAndPayload));
clear();
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 4a7fef11af49b..e86651523167d 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -34,6 +34,8 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -1416,7 +1418,9 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien
private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId,
MessageIdData messageId, ClientCnx cnx) {
-
+ if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) {
+ increaseAvailablePermits(cnx);
+ }
// Lazy task scheduling to expire incomplete chunk message
if (!expireChunkMessageTaskScheduled && expireTimeOfIncompleteChunkedMessageMillis > 0) {
((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).scheduleAtFixedRate(
@@ -1428,11 +1432,58 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m
expireChunkMessageTaskScheduled = true;
}
+ ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid());
+
if (msgMetadata.getChunkId() == 0) {
+ if (chunkedMsgCtx != null) {
+ // Handle ack hole case when receive duplicated chunks.
+ // There are two situation that receives chunks with the same sequence ID and chunk ID.
+ // Situation 1 - Message redeliver:
+ // For example:
+ // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1
+ // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2
+ // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:1
+ // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:2
+ // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:3
+ // In this case, chunk-3 and chunk-4 have the same msgID with chunk-1 and chunk-2.
+ // This may be caused by message redeliver, we can't ack any chunk in this case here.
+ // Situation 2 - Corrupted chunk message
+ // For example:
+ // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1
+ // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2
+ // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:3
+ // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4
+ // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5
+ // In this case, all the chunks with different msgIDs and are persistent in the topic.
+ // But Chunk-1 and Chunk-2 belong to a corrupted chunk message that must be skipped since
+ // they will not be delivered to end users. So we should ack them here to avoid ack hole.
+ boolean isCorruptedChunkMessageDetected = Arrays.stream(chunkedMsgCtx.chunkedMessageIds)
+ .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId()
+ && messageId1.entryId == messageId.getEntryId());
+ if (isCorruptedChunkMessageDetected) {
+ Arrays.stream(chunkedMsgCtx.chunkedMessageIds).forEach(messageId1 -> {
+ if (messageId1 != null) {
+ doAcknowledge(messageId1, AckType.Individual, Collections.emptyMap(), null);
+ }
+ });
+ }
+ // The first chunk of a new chunked-message received before receiving other chunks of previous
+ // chunked-message
+ // so, remove previous chunked-message from map and release buffer
+ if (chunkedMsgCtx.chunkedMsgBuffer != null) {
+ ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer);
+ }
+ chunkedMsgCtx.recycle();
+ chunkedMessagesMap.remove(msgMetadata.getUuid());
+ }
+ pendingChunkedMessageCount++;
+ if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) {
+ removeOldestPendingChunkedMessage();
+ }
+ int totalChunks = msgMetadata.getNumChunksFromMsg();
ByteBuf chunkedMsgBuffer = PulsarByteBufAllocator.DEFAULT.buffer(msgMetadata.getTotalChunkMsgSize(),
msgMetadata.getTotalChunkMsgSize());
- int totalChunks = msgMetadata.getNumChunksFromMsg();
- chunkedMessagesMap.computeIfAbsent(msgMetadata.getUuid(),
+ chunkedMsgCtx = chunkedMessagesMap.computeIfAbsent(msgMetadata.getUuid(),
(key) -> ChunkedMessageCtx.get(totalChunks, chunkedMsgBuffer));
pendingChunkedMessageCount++;
if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) {
@@ -1441,12 +1492,37 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m
pendingChunkedMessageUuidQueue.add(msgMetadata.getUuid());
}
- ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid());
// discard message if chunk is out-of-order
if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null
|| msgMetadata.getChunkId() != (chunkedMsgCtx.lastChunkedMessageId + 1)) {
+ // Filter and ack duplicated chunks instead of discard ctx.
+ // For example:
+ // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1
+ // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2
+ // Chunk-3 sequence ID: 0, chunk ID: 2, msgID: 1:3
+ // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4
+ // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5
+ // Chunk-6 sequence ID: 0, chunk ID: 3, msgID: 1:6
+ // We should filter and ack chunk-4 and chunk-5.
+ if (chunkedMsgCtx != null && msgMetadata.getChunkId() <= chunkedMsgCtx.lastChunkedMessageId) {
+ log.warn("[{}] Receive a duplicated chunk message with messageId [{}], last-chunk-Id [{}], "
+ + "chunkId [{}], sequenceId [{}]",
+ msgMetadata.getProducerName(), msgId, chunkedMsgCtx.lastChunkedMessageId,
+ msgMetadata.getChunkId(), msgMetadata.getSequenceId());
+ compressedPayload.release();
+ // Just like the above logic of receiving the first chunk again. We only ack this chunk in the message
+ // duplication case.
+ boolean isDuplicatedChunk = Arrays.stream(chunkedMsgCtx.chunkedMessageIds)
+ .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId()
+ && messageId1.entryId == messageId.getEntryId());
+ if (isDuplicatedChunk) {
+ doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null);
+ }
+ return null;
+ }
// means we lost the first chunk: should never happen
- log.info("Received unexpected chunk messageId {}, last-chunk-id{}, chunkId = {}", msgId,
+ log.info("[{}] [{}] Received unexpected chunk messageId {}, last-chunk-id = {}, chunkId = {}", topic,
+ subscription, msgId,
(chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), msgMetadata.getChunkId());
if (chunkedMsgCtx != null) {
if (chunkedMsgCtx.chunkedMsgBuffer != null) {
@@ -1456,7 +1532,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m
}
chunkedMessagesMap.remove(msgMetadata.getUuid());
compressedPayload.release();
- increaseAvailablePermits(cnx);
if (expireTimeOfIncompleteChunkedMessageMillis > 0
&& System.currentTimeMillis() > (msgMetadata.getPublishTime()
+ expireTimeOfIncompleteChunkedMessageMillis)) {
@@ -1475,7 +1550,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m
// if final chunk is not received yet then release payload and return
if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) {
compressedPayload.release();
- increaseAvailablePermits(cnx);
return null;
}
diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java
index 56443709bf354..954375919d80b 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java
@@ -167,6 +167,8 @@ public CompletableFuture connectionOpened(ClientCnx cnx) {
return null;
});
} else {
+ LOG.warn("Can not connect to the transaction coordinator because the protocol version {} is "
+ + "lower than 19", cnx.getRemoteEndpointProtocolVersion());
registerToConnection(cnx);
future.complete(null);
}
diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml
index a37c886466ea4..121204d24e564 100644
--- a/pulsar-client/src/main/resources/findbugsExclude.xml
+++ b/pulsar-client/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml
index 08889451dec31..580146001b4f5 100644
--- a/pulsar-common/pom.xml
+++ b/pulsar-common/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-common
Pulsar Common
Common libraries needed by client/broker/tools
-
${project.groupId}
pulsar-client-api
${project.version}
-
${project.groupId}
pulsar-client-admin-api
${project.version}
-
io.swagger
swagger-annotations
-
org.slf4j
slf4j-api
-
- com.fasterxml.jackson.core
- jackson-databind
+ com.fasterxml.jackson.core
+ jackson-databind
-
com.google.guava
guava
-
io.netty
netty-handler
-
io.netty
netty-resolver-dns
-
io.netty
netty-transport-native-epoll
linux-x86_64
-
io.netty
netty-transport-native-unix-common
linux-x86_64
-
org.apache.bookkeeper
bookkeeper-common-allocator
@@ -99,17 +85,14 @@
-
org.apache.bookkeeper
cpu-affinity
-
io.airlift
aircompressor
-
org.apache.bookkeeper
circe-checksum
@@ -125,44 +108,36 @@
-
io.netty
netty-tcnative-boringssl-static
-
io.netty
netty-codec-haproxy
-
org.apache.commons
commons-lang3
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
javax.ws.rs
javax.ws.rs-api
-
commons-io
commons-io
-
com.github.spotbugs
spotbugs-annotations
provided
true
-
org.bouncycastle
@@ -170,78 +145,70 @@
${bouncycastlefips.version}
test
-
org.lz4
lz4-java
1.5.0
test
-
com.github.luben
zstd-jni
1.3.7-3
test
-
org.xerial.snappy
snappy-java
1.1.7.5
test
-
- com.google.code.gson
- gson
-
-
+
+ com.google.code.gson
+ gson
+
com.beust
jcommander
-
org.awaitility
awaitility
test
-
-
-
- org.gaul
- modernizer-maven-plugin
-
- true
- 8
-
-
-
- modernizer
- verify
-
- modernizer
-
-
-
-
-
-
- com.github.splunk.lightproto
- lightproto-maven-plugin
- ${lightproto-maven-plugin.version}
-
-
-
- generate
-
-
-
-
-
-
+
+
+ org.gaul
+ modernizer-maven-plugin
+
+ true
+ 8
+
+
+
+ modernizer
+ verify
+
+ modernizer
+
+
+
+
+
+ com.github.splunk.lightproto
+ lightproto-maven-plugin
+ ${lightproto-maven-plugin.version}
+
+
+
+ generate
+
+
+
+
+
pl.project13.maven
git-commit-id-plugin
@@ -263,51 +230,50 @@
true
-
-
-
- org.codehaus.mojo
- templating-maven-plugin
- 1.0.0
-
-
- filtering-java-templates
-
- filter-sources
-
-
-
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
-
- spotbugs
- verify
-
- check
-
-
-
-
- ${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- checkstyle
- verify
-
- check
-
-
-
-
-
+
+
+ org.codehaus.mojo
+ templating-maven-plugin
+ 1.0.0
+
+
+ filtering-java-templates
+
+ filter-sources
+
+
+
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+
+ spotbugs
+ verify
+
+ check
+
+
+
+
+ ${basedir}/src/main/resources/findbugsExclude.xml
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ checkstyle
+ verify
+
+ check
+
+
+
+
+
diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java
new file mode 100644
index 0000000000000..024aa153756dc
--- /dev/null
+++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.util;
+
+import java.util.function.Supplier;
+
+/***
+ * Used to lazy load a value, only calculate it when used. Not thread-safety.
+ */
+public class LazyLoadableValue {
+
+ private Supplier loader;
+
+ private T value;
+
+ public LazyLoadableValue(Supplier loader) {
+ this.loader = loader;
+ }
+
+ public T getValue() {
+ if (value == null) {
+ value = loader.get();
+ }
+ return value;
+ }
+}
diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml
index df161c4b621a7..8216948c12408 100644
--- a/pulsar-common/src/main/resources/findbugsExclude.xml
+++ b/pulsar-common/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml
index 1df8526d8a02d..5258bc4c64219 100644
--- a/pulsar-config-validation/pom.xml
+++ b/pulsar-config-validation/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
- org.apache.pulsar
- pulsar
- 2.10.5-SNAPSHOT
- ..
-
-
- pulsar-config-validation
- Annotation based config validation for Pulsar
-
-
-
- org.slf4j
- slf4j-api
-
-
-
-
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- checkstyle
- verify
-
- check
-
-
-
-
-
-
-
\ No newline at end of file
+
+ 4.0.0
+
+ io.streamnative
+ pulsar
+ 2.10.5.3
+ ..
+
+ pulsar-config-validation
+ Annotation based config validation for Pulsar
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+
+
+ checkstyle
+ verify
+
+ check
+
+
+
+
+
+
+
diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml
index 47097127dfb80..3b69ddbe1f519 100644
--- a/pulsar-functions/api-java/pom.xml
+++ b/pulsar-functions/api-java/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-api
Pulsar Functions :: API
-
-
org.slf4j
slf4j-api
-
net.jodah
typetools
@@ -48,16 +44,13 @@
${project.version}
compile
-
${project.groupId}
pulsar-client-admin-api
${project.version}
compile
-
-
@@ -79,5 +72,4 @@
-
diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml
index e984ca282c372..da55998594007 100644
--- a/pulsar-functions/instance/pom.xml
+++ b/pulsar-functions/instance/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-instance
Pulsar Functions :: Instance
-
-
org.apache.logging.log4j
@@ -46,61 +42,51 @@
org.apache.logging.log4j
log4j-core
-
${project.groupId}
pulsar-functions-utils
${project.version}
-
${project.groupId}
pulsar-metadata
${project.version}
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-io-common
${project.version}
-
${project.groupId}
pulsar-functions-api
${project.version}
-
${project.groupId}
pulsar-functions-secrets
${project.version}
-
-
+
${project.groupId}
pulsar-client-admin-original
${project.version}
-
-
+
${project.groupId}
pulsar-client-original
${project.version}
-
${project.groupId}
pulsar-client-messagecrypto-bc
${project.parent.version}
-
org.apache.bookkeeper
stream-storage-java-client
@@ -115,24 +101,20 @@
-
io.grpc
grpc-stub
-
io.grpc
grpc-all
-
io.perfmark
perfmark-api
runtime
-
org.apache.bookkeeper
bookkeeper-common
@@ -147,61 +129,50 @@
-
com.yahoo.datasketches
sketches-core
-
com.google.guava
guava
-
com.beust
jcommander
-
net.jodah
typetools
-
io.prometheus
simpleclient
${prometheus.version}
-
io.prometheus
simpleclient_hotspot
${prometheus.version}
-
io.prometheus
simpleclient_httpserver
${prometheus.version}
-
io.prometheus.jmx
collector
${prometheus-jmx.version}
-
org.awaitility
awaitility
test
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml
index 7283d60ffff9e..84aaadad8e5a4 100644
--- a/pulsar-functions/java-examples/pom.xml
+++ b/pulsar-functions/java-examples/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-api-examples
Pulsar Functions :: API Examples
-
org.slf4j
slf4j-api
-
${project.groupId}
pulsar-functions-api
@@ -52,7 +49,6 @@
${project.version}
-
@@ -74,5 +70,4 @@
-
diff --git a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml
index abaf77f45a45e..67a501bf80c63 100644
--- a/pulsar-functions/localrun-shaded/pom.xml
+++ b/pulsar-functions/localrun-shaded/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar-functions
- 2.10.5-SNAPSHOT
- ..
-
-
- pulsar-functions-local-runner
- Pulsar Functions :: Local Runner Shaded
-
-
-
- ${project.groupId}
- pulsar-functions-local-runner-original
- ${project.parent.version}
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-dependency-plugin
-
-
- unpack
- prepare-package
-
- unpack
-
-
-
-
- org.asynchttpclient
- async-http-client
- ${asynchttpclient.version}
- jar
- true
- org/asynchttpclient/config/ahc-default.properties
- ${project.build.directory}/classes
-
-
-
-
-
-
-
-
- maven-antrun-plugin
-
-
- shade-ahc-properties
- prepare-package
-
- run
-
-
-
-
-
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
-
-
- ${shadePluginPhase}
-
- shade
-
-
- false
-
-
-
-
-
-
- org.apache.pulsar:*
- org.apache.bookkeeper:*
- commons-*:*
- org.apache.commons:*
- com.fasterxml.jackson.*:*
- io.netty:*
- com.google.*:*
- javax.servlet:*
- org.reactivestreams:reactive-streams
- org.apache.commons:*
- io.swagger:*
- org.yaml:snakeyaml
- io.perfmark:*
- io.prometheus:*
- io.prometheus.jmx:*
- javax.ws.rs:*
- org.tukaani:xz
- com.github.zafarkhaja:java-semver
- net.java.dev.jna:*
- org.apache.zookeeper:*
- com.thoughtworks.paranamer:paranamer
- jline:*
- org.rocksdb:*
- org.eclipse.jetty*:*
- org.apache.avro:avro
- com.beust:*
- net.jodah:*
- io.airlift:*
- com.yahoo.datasketches:*
- io.netty.resolver:*
-
-
-
-
- org.apache.pulsar:pulsar-client-original
-
- **
-
-
-
- org/bouncycastle/**
-
-
-
-
-
-
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ ${shadePluginPhase}
+
+ shade
+
+
+ false
+
+
+
+
+
+
+ io.streamnative:*
+ org.apache.bookkeeper:*
+ commons-*:*
+ org.apache.commons:*
+ com.fasterxml.jackson.*:*
+ io.netty:*
+ com.google.*:*
+ javax.servlet:*
+ org.reactivestreams:reactive-streams
+ org.apache.commons:*
+ io.swagger:*
+ org.yaml:snakeyaml
+ io.perfmark:*
+ io.prometheus:*
+ io.prometheus.jmx:*
+ javax.ws.rs:*
+ org.tukaani:xz
+ com.github.zafarkhaja:java-semver
+ net.java.dev.jna:*
+ org.apache.zookeeper:*
+ com.thoughtworks.paranamer:paranamer
+ jline:*
+ org.rocksdb:*
+ org.eclipse.jetty*:*
+ org.apache.avro:avro
+ com.beust:*
+ net.jodah:*
+ io.airlift:*
+ com.yahoo.datasketches:*
+ io.netty.resolver:*
+
+
+
+
+ io.streamnative:pulsar-client-original
+
+ **
+
+
+
+ org/bouncycastle/**
+
+
+
+
+
+
-
- com.google
- org.apache.pulsar.functions.runtime.shaded.com.google
-
-
- org.apache.jute
- org.apache.pulsar.functions.runtime.shaded.org.apache.jute
-
-
- javax.servlet
- org.apache.pulsar.functions.runtime.shaded.javax.servlet
-
-
- org.junit
- org.apache.pulsar.functions.runtime.shaded.org.junit
-
-
- junit
- org.apache.pulsar.functions.runtime.shaded.junit
-
-
- net.jodah
- org.apache.pulsar.functions.runtime.shaded.net.jodah
-
-
- org.lz4
- org.apache.pulsar.functions.runtime.shaded.org.lz4
-
-
- org.reactivestreams
- org.apache.pulsar.functions.runtime.shaded.org.reactivestreams
-
-
- org.apache.commons
- org.apache.pulsar.functions.runtime.shaded.org.apache.commons
-
-
- io.swagger
- org.apache.pulsar.functions.runtime.shaded.io.swagger
-
-
- org.yaml
- org.apache.pulsar.functions.runtime.shaded.org.yaml
-
-
- org.jctools
- org.apache.pulsar.functions.runtime.shaded.org.jctools
-
-
- com.squareup.okhttp
- org.apache.pulsar.functions.runtime.shaded.com.squareup.okhttp
-
-
- io.grpc
- org.apache.pulsar.functions.runtime.shaded.io.grpc
-
-
- io.perfmark
- org.apache.pulsar.functions.runtime.shaded.io.perfmark
-
-
- org.joda
- org.apache.pulsar.functions.runtime.shaded.org.joda
-
-
- javax.ws.rs
- org.apache.pulsar.functions.runtime.shaded.javax.ws.rs
-
-
- io.kubernetes
- org.apache.pulsar.functions.runtime.shaded.io.kubernetes
-
-
- io.opencensus
- org.apache.pulsar.functions.runtime.shaded.io.opencensus
-
-
- net.jpountz
- org.apache.pulsar.functions.runtime.shaded.net.jpountz
-
-
- commons-configuration
- org.apache.pulsar.functions.runtime.shaded.commons-configuration
-
-
- org.tukaani
- org.apache.pulsar.functions.runtime.shaded.org.tukaani
-
-
- com.github
- org.apache.pulsar.functions.runtime.shaded.com.github
-
-
- commons-io
- org.apache.pulsar.functions.runtime.shaded.commons-io
-
-
- org.apache.distributedlog
- org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog
-
-
-
- com.fasterxml
- org.apache.pulsar.functions.runtime.shaded.com.fasterxml
-
-
- org.inferred
- org.apache.pulsar.functions.runtime.shaded.org.inferred
-
-
- org.apache.bookkeeper
- org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
-
-
- org.bookkeeper
- org.apache.pulsar.functions.runtime.shaded.org.bookkeeper
-
-
- dlshade
- org.apache.pulsar.functions.runtime.shaded.dlshade
-
-
-
- org.codehaus.jackson
- org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson
-
-
- net.java.dev.jna
- org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna
-
-
- org.apache.curator
- org.apache.pulsar.functions.runtime.shaded.org.apache.curator
-
-
- javax.validation
- org.apache.pulsar.functions.runtime.shaded.javax.validation
-
-
- javax.activation
- org.apache.pulsar.functions.runtime.shaded.javax.activation
-
-
- io.prometheus
- org.apache.pulsar.functions.runtime.shaded.io.prometheus
-
-
- io.prometheus.jmx
- org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx
-
-
- org.apache.zookeeper
- org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper
-
-
- io.jsonwebtoken
- org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken
-
-
- commons-codec
- org.apache.pulsar.functions.runtime.shaded.commons-codec
-
-
- com.thoughtworks.paranamer
- org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer
-
-
- org.codehaus.mojo
- org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo
-
-
- com.github.luben
- org.apache.pulsar.functions.runtime.shaded.com.github.luben
-
-
- jline
- org.apache.pulsar.functions.runtime.shaded.jline
-
-
- org.xerial.snappy
- org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy
-
-
- javax.annotation
- org.apache.pulsar.functions.runtime.shaded.javax.annotation
-
-
- org.checkerframework
- org.apache.pulsar.functions.runtime.shaded.org.checkerframework
-
-
- org.apache.yetus
- org.apache.pulsar.functions.runtime.shaded.org.apache.yetus
-
-
- commons-cli
- org.apache.pulsar.functions.runtime.shaded.commons-cli
-
-
- commons-lang
- org.apache.pulsar.functions.runtime.shaded.commons-lang
-
-
- com.squareup.okio
- org.apache.pulsar.functions.runtime.shaded.com.squareup.okio
-
-
- org.rocksdb
- org.apache.pulsar.functions.runtime.shaded.org.rocksdb
-
-
- org.objenesis
- org.apache.pulsar.functions.runtime.shaded.org.objenesis
-
-
- org.eclipse.jetty
- org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty
-
-
- org.apache.avro
- org.apache.pulsar.functions.runtime.shaded.org.apache.avro
-
-
- avro.shaded
- org.apache.pulsar.functions.runtime.shaded.avo.shaded
-
-
- com.yahoo.datasketches
- org.apache.pulsar.shaded.com.yahoo.datasketches
-
-
- com.yahoo.sketches
- org.apache.pulsar.shaded.com.yahoo.sketches
-
-
- com.beust
- org.apache.pulsar.functions.runtime.shaded.com.beust
-
-
-
- io.netty
- org.apache.pulsar.functions.runtime.shaded.io.netty
-
-
- org.hamcrest
- org.apache.pulsar.functions.runtime.shaded.org.hamcrest
-
-
- aj.org
- org.apache.pulsar.functions.runtime.shaded.aj.org
-
-
- com.scurrilous
- org.apache.pulsar.functions.runtime.shaded.com.scurrilous
-
-
- okio
- org.apache.pulsar.functions.runtime.shaded.okio
-
-
+
+ com.fasterxml
+ org.apache.pulsar.functions.runtime.shaded.com.fasterxml
+
+
+ org.inferred
+ org.apache.pulsar.functions.runtime.shaded.org.inferred
+
+
+ org.apache.bookkeeper
+ org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
+
+
+ org.bookkeeper
+ org.apache.pulsar.functions.runtime.shaded.org.bookkeeper
+
+
+ dlshade
+ org.apache.pulsar.functions.runtime.shaded.dlshade
+
+
+
+ org.codehaus.jackson
+ org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson
+
+
+ net.java.dev.jna
+ org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna
+
+
+ org.apache.curator
+ org.apache.pulsar.functions.runtime.shaded.org.apache.curator
+
+
+ javax.validation
+ org.apache.pulsar.functions.runtime.shaded.javax.validation
+
+
+ javax.activation
+ org.apache.pulsar.functions.runtime.shaded.javax.activation
+
+
+ io.prometheus
+ org.apache.pulsar.functions.runtime.shaded.io.prometheus
+
+
+ io.prometheus.jmx
+ org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx
+
+
+ org.apache.zookeeper
+ org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper
+
+
+ io.jsonwebtoken
+ org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken
+
+
+ commons-codec
+ org.apache.pulsar.functions.runtime.shaded.commons-codec
+
+
+ com.thoughtworks.paranamer
+ org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer
+
+
+ org.codehaus.mojo
+ org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo
+
+
+ com.github.luben
+ org.apache.pulsar.functions.runtime.shaded.com.github.luben
+
+
+ jline
+ org.apache.pulsar.functions.runtime.shaded.jline
+
+
+ org.xerial.snappy
+ org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy
+
+
+ javax.annotation
+ org.apache.pulsar.functions.runtime.shaded.javax.annotation
+
+
+ org.checkerframework
+ org.apache.pulsar.functions.runtime.shaded.org.checkerframework
+
+
+ org.apache.yetus
+ org.apache.pulsar.functions.runtime.shaded.org.apache.yetus
+
+
+ commons-cli
+ org.apache.pulsar.functions.runtime.shaded.commons-cli
+
+
+ commons-lang
+ org.apache.pulsar.functions.runtime.shaded.commons-lang
+
+
+ com.squareup.okio
+ org.apache.pulsar.functions.runtime.shaded.com.squareup.okio
+
+
+ org.rocksdb
+ org.apache.pulsar.functions.runtime.shaded.org.rocksdb
+
+
+ org.objenesis
+ org.apache.pulsar.functions.runtime.shaded.org.objenesis
+
+
+ org.eclipse.jetty
+ org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty
+
+
+ org.apache.avro
+ org.apache.pulsar.functions.runtime.shaded.org.apache.avro
+
+
+ avro.shaded
+ org.apache.pulsar.functions.runtime.shaded.avo.shaded
+
+
+ com.yahoo.datasketches
+ org.apache.pulsar.shaded.com.yahoo.datasketches
+
+
+ com.yahoo.sketches
+ org.apache.pulsar.shaded.com.yahoo.sketches
+
+
+ com.beust
+ org.apache.pulsar.functions.runtime.shaded.com.beust
+
+
+
+ io.netty
+ org.apache.pulsar.functions.runtime.shaded.io.netty
+
+
+ org.hamcrest
+ org.apache.pulsar.functions.runtime.shaded.org.hamcrest
+
+
+ aj.org
+ org.apache.pulsar.functions.runtime.shaded.aj.org
+
+
+ com.scurrilous
+ org.apache.pulsar.functions.runtime.shaded.com.scurrilous
+
+
+ okio
+ org.apache.pulsar.functions.runtime.shaded.okio
+
+
-
- org.asynchttpclient
- org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient
-
-
- io.airlift
- org.apache.pulsar.functions.runtime.shaded.io.airlift
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
+
+
diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml
index 2744b5aa51c75..5552e412e7313 100644
--- a/pulsar-functions/localrun/pom.xml
+++ b/pulsar-functions/localrun/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar-functions
- 2.10.5-SNAPSHOT
- ..
-
-
- pulsar-functions-local-runner-original
- Pulsar Functions :: Local Runner original
-
-
-
- ${project.groupId}
- pulsar-functions-runtime
- ${project.parent.version}
-
-
-
-
- org.apache.logging.log4j
- log4j-slf4j-impl
-
-
- org.apache.logging.log4j
- log4j-api
-
-
- org.apache.logging.log4j
- log4j-core
-
-
-
- io.grpc
- grpc-all
-
-
-
- io.perfmark
- perfmark-api
- runtime
-
-
-
-
-
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
- ${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
- check
- verify
-
- check
-
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-functions
+ 2.10.5.3
+ ..
+
+ pulsar-functions-local-runner-original
+ Pulsar Functions :: Local Runner original
+
+
+ ${project.groupId}
+ pulsar-functions-runtime
+ ${project.parent.version}
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+
+
+ org.apache.logging.log4j
+ log4j-api
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+ io.grpc
+ grpc-all
+
+
+ io.perfmark
+ perfmark-api
+ runtime
+
+
+
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+ ${basedir}/src/main/resources/findbugsExclude.xml
+
+
+
+ check
+ verify
+
+ check
+
+
+
+
+
+
diff --git a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml
index 719b711d6560a..02157e2e9ea6f 100644
--- a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
-
-
-
\ No newline at end of file
+
+
+
+
+
diff --git a/pulsar-functions/localrun/src/main/resources/log4j2.xml b/pulsar-functions/localrun/src/main/resources/log4j2.xml
index a7714c3f48835..aa28dda160b02 100644
--- a/pulsar-functions/localrun/src/main/resources/log4j2.xml
+++ b/pulsar-functions/localrun/src/main/resources/log4j2.xml
@@ -1,4 +1,4 @@
-
+
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
+
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml
index 02bb1210b092e..3f4119dcde70b 100644
--- a/pulsar-functions/pom.xml
+++ b/pulsar-functions/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
pom
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions
Pulsar Functions :: Parent
-
main
@@ -41,23 +39,22 @@
!true
-
- proto
- api-java
- java-examples
- utils
- instance
- runtime
- runtime-all
- worker
- secrets
- localrun
- localrun-shaded
-
+
+ proto
+ api-java
+ java-examples
+ utils
+ instance
+ runtime
+ runtime-all
+ worker
+ secrets
+ localrun
+ localrun-shaded
+
-
-
- core-modules
+
+ core-modules
proto
api-java
@@ -70,8 +67,6 @@
secrets
localrun
-
-
+
-
diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml
index beb1ab2d5be93..6a05246337f4d 100644
--- a/pulsar-functions/proto/pom.xml
+++ b/pulsar-functions/proto/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar-functions
- 2.10.5-SNAPSHOT
-
-
- pulsar-functions-proto
- Pulsar Functions :: Proto
-
-
-
-
- com.google.protobuf
- protobuf-java
-
-
-
- com.google.protobuf
- protobuf-java-util
-
-
-
- javax.annotation
- javax.annotation-api
-
-
-
- io.grpc
- grpc-all
-
-
- io.netty
- *
-
-
-
-
-
- io.perfmark
- perfmark-api
- runtime
-
-
-
-
-
-
- org.xolstice.maven.plugins
- protobuf-maven-plugin
- ${protobuf-maven-plugin.version}
-
- com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier}
- true
- grpc-java
- io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier}
-
-
-
- generate-sources
-
- compile
- compile-custom
-
-
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-functions
+ 2.10.5.3
+
+ pulsar-functions-proto
+ Pulsar Functions :: Proto
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+ com.google.protobuf
+ protobuf-java-util
+
+
+ javax.annotation
+ javax.annotation-api
+
+
+ io.grpc
+ grpc-all
+
+
+ io.netty
+ *
+
+
+
+
+ io.perfmark
+ perfmark-api
+ runtime
+
+
+
+
+
+ org.xolstice.maven.plugins
+ protobuf-maven-plugin
+ ${protobuf-maven-plugin.version}
+
+ com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier}
+ true
+ grpc-java
+ io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier}
+
+
+
+ generate-sources
+
+ compile
+ compile-custom
+
+
+
+
+
+
diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml
index 06bede1abfb5e..519785ecac698 100644
--- a/pulsar-functions/runtime-all/pom.xml
+++ b/pulsar-functions/runtime-all/pom.xml
@@ -1,4 +1,4 @@
-
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
-
pulsar-functions-runtime-all
Pulsar Functions :: Runtime All
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-functions-api
${project.version}
-
${project.groupId}
pulsar-client-api
${project.version}
-
org.apache.avro
avro
${avro.version}
-
com.fasterxml.jackson.core
jackson-databind
-
com.google.protobuf
@@ -94,30 +84,24 @@
gson
${gson.version}
-
-
org.slf4j
slf4j-api
-
org.apache.logging.log4j
log4j-slf4j-impl
-
org.apache.logging.log4j
log4j-api
-
org.apache.logging.log4j
log4j-core
-
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml
index 190d9be92940b..84e6ce44f51e4 100644
--- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml
+++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml
@@ -1,3 +1,4 @@
+
- pulsar-functions-instance
- 30
-
-
- pulsar.log.appender
- RollingFile
-
-
- pulsar.log.level
- info
-
-
- bk.log.level
- info
-
-
-
-
- Console
- SYSTEM_OUT
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- RollingFile
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz
- true
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- 1
- true
-
-
- 1 GB
-
-
- 0 0 0 * * ?
-
-
-
-
- ${sys:pulsar.function.log.dir}
- 2
-
- */${sys:pulsar.function.log.file}*log.gz
-
-
- 30d
-
-
-
-
-
- BkRollingFile
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk
- ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz
- true
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
- 1
- true
-
-
- 1 GB
-
-
- 0 0 0 * * ?
-
-
-
-
- ${sys:pulsar.function.log.dir}
- 2
-
- */${sys:pulsar.function.log.file}.bk*log.gz
-
-
- 30d
-
-
-
-
-
-
-
- org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
- ${sys:bk.log.level}
- false
-
- [BkRollingFile]
-
-
-
- info
-
- [${sys:pulsar.log.appender}]
- ${sys:pulsar.log.level}
-
-
-
-
\ No newline at end of file
+ pulsar-functions-instance
+ 30
+
+
+ pulsar.log.appender
+ RollingFile
+
+
+ pulsar.log.level
+ info
+
+
+ bk.log.level
+ info
+
+
+
+
+ Console
+ SYSTEM_OUT
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ RollingFile
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz
+ true
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ 1
+ true
+
+
+ 1 GB
+
+
+ 0 0 0 * * ?
+
+
+
+
+ ${sys:pulsar.function.log.dir}
+ 2
+
+ */${sys:pulsar.function.log.file}*log.gz
+
+
+ 30d
+
+
+
+
+
+ BkRollingFile
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk
+ ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz
+ true
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+ 1
+ true
+
+
+ 1 GB
+
+
+ 0 0 0 * * ?
+
+
+
+
+ ${sys:pulsar.function.log.dir}
+ 2
+
+ */${sys:pulsar.function.log.file}.bk*log.gz
+
+
+ 30d
+
+
+
+
+
+
+
+ org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
+ ${sys:bk.log.level}
+ false
+
+ [BkRollingFile]
+
+
+
+ info
+
+ [${sys:pulsar.log.appender}]
+ ${sys:pulsar.log.level}
+
+
+
+
diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml
index f86d03e41793f..fe478be5b8667 100644
--- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml
+++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml
@@ -1,3 +1,4 @@
+
- pulsar-functions-kubernetes-instance
- 30
-
-
- pulsar.log.level
- info
-
-
- bk.log.level
- info
-
-
-
-
- Console
- SYSTEM_OUT
-
- %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
-
-
-
-
-
- org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
- ${sys:bk.log.level}
- false
-
- [Console]
-
-
-
- info
-
- [Console]
- ${sys:pulsar.log.level}
-
-
-
-
\ No newline at end of file
+ pulsar-functions-kubernetes-instance
+ 30
+
+
+ pulsar.log.level
+ info
+
+
+ bk.log.level
+ info
+
+
+
+
+ Console
+ SYSTEM_OUT
+
+ %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n
+
+
+
+
+
+ org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper
+ ${sys:bk.log.level}
+ false
+
+ [Console]
+
+
+
+ info
+
+ [Console]
+ ${sys:pulsar.log.level}
+
+
+
+
diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml
index afc7b73c07cf7..dc0f3ab3d5ff0 100644
--- a/pulsar-functions/runtime/pom.xml
+++ b/pulsar-functions/runtime/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-runtime
Pulsar Functions :: Runtime
-
-
${project.groupId}
pulsar-functions-instance
${project.version}
-
${project.groupId}
pulsar-functions-secrets
${project.version}
-
com.beust
jcommander
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.fasterxml.jackson.core
jackson-databind
-
io.kubernetes
client-java
@@ -70,9 +61,7 @@
pulsar-broker-common
${project.version}
-
-
@@ -118,5 +107,4 @@
-
diff --git a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml
index b4a4e5926de3f..976abe7c54bb2 100644
--- a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
+
+
+
+
+
+
+
+
+
+
diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml
index 7712da5ffbc71..f3a54c847b9c9 100644
--- a/pulsar-functions/secrets/pom.xml
+++ b/pulsar-functions/secrets/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-secrets
Pulsar Functions :: Secrets
-
io.kubernetes
@@ -50,7 +48,6 @@
-
${project.groupId}
pulsar-functions-proto
@@ -78,5 +75,4 @@
-
diff --git a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml
index 1ccdd44f56161..aa6cf7036e6db 100644
--- a/pulsar-functions/utils/pom.xml
+++ b/pulsar-functions/utils/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-functions-utils
Pulsar Functions :: Utils
-
${project.groupId}
pulsar-common
${project.version}
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-functions-proto
${project.version}
-
${project.groupId}
pulsar-functions-api
${project.version}
-
${project.groupId}
pulsar-config-validation
${project.version}
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.fasterxml.jackson.core
jackson-annotations
-
com.fasterxml.jackson.core
jackson-databind
-
com.google.code.gson
gson
-
net.jodah
typetools
-
- ${project.groupId}
- pulsar-client-original
- ${project.version}
+ ${project.groupId}
+ pulsar-client-original
+ ${project.version}
-
${project.groupId}
pulsar-package-core
${project.version}
-
-
diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml
index 31debbdf14e02..b2f3beff76803 100644
--- a/pulsar-functions/worker/pom.xml
+++ b/pulsar-functions/worker/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-functions
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
pulsar-functions-worker
Pulsar Functions :: Worker
-
-
${project.groupId}
pulsar-broker-common
${project.version}
-
${project.groupId}
pulsar-functions-runtime
${project.version}
-
${project.groupId}
pulsar-client-original
${project.version}
-
${project.groupId}
pulsar-client-admin-original
${project.version}
-
org.glassfish.jersey.media
jersey-media-json-jackson
-
jakarta.activation
jakarta.activation-api
-
org.glassfish.jersey.core
jersey-server
-
org.glassfish.jersey.containers
jersey-container-servlet
-
org.glassfish.jersey.containers
jersey-container-servlet-core
-
org.glassfish.jersey.media
jersey-media-multipart
-
org.eclipse.jetty
jetty-server
-
org.eclipse.jetty
jetty-alpn-conscrypt-server
-
org.eclipse.jetty
jetty-servlet
-
org.apache.distributedlog
distributedlog-core
@@ -113,17 +96,14 @@
-
org.apache.bookkeeper
bookkeeper-server
-
commons-io
commons-io
-
io.swagger
swagger-core
@@ -138,14 +118,11 @@
-
io.prometheus
simpleclient_jetty
-
-
${project.groupId}
pulsar-io-cassandra
@@ -153,7 +130,6 @@
pom
test
-
${project.groupId}
pulsar-io-twitter
@@ -161,7 +137,6 @@
pom
test
-
${project.groupId}
pulsar-functions-api-examples
@@ -169,9 +144,7 @@
pom
test
-
-
@@ -217,7 +190,6 @@
-
org.apache.maven.plugins
maven-surefire-plugin
@@ -232,7 +204,6 @@
-
org.apache.maven.plugins
@@ -264,17 +235,16 @@
${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-
diff --git a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml
+++ b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml
index df6e964c02483..5406c1868bad6 100644
--- a/pulsar-io/aerospike/pom.xml
+++ b/pulsar-io/aerospike/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-aerospike
Pulsar IO :: Aerospike
-
-
${project.groupId}
pulsar-io-core
${project.version}
-
com.fasterxml.jackson.core
jackson-databind
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.aerospike
aerospike-client-bc
${aerospike-client.version}
-
-
@@ -81,6 +73,4 @@
-
-
diff --git a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml
index 634ed92122bab..ad3aa3adcbadf 100644
--- a/pulsar-io/aws/pom.xml
+++ b/pulsar-io/aws/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-aws
Pulsar IO :: IO AWS
-
${project.groupId}
pulsar-io-core
${project.version}
-
com.google.code.gson
gson
-
com.amazonaws
aws-java-sdk-sts
-
software.amazon.awssdk
sts
2.10.56
-
-
+
-
diff --git a/pulsar-io/aws/src/main/resources/findbugsExclude.xml b/pulsar-io/aws/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-io/aws/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/aws/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml
index b191f2be714d1..99fd1389abe9e 100644
--- a/pulsar-io/batch-data-generator/pom.xml
+++ b/pulsar-io/batch-data-generator/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- org.apache.pulsar
- pulsar-io
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-batch-data-generator
- Pulsar IO :: Batch Data Generator
-
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
-
- ${project.groupId}
- pulsar-io-batch-discovery-triggerers
- ${project.version}
-
-
-
- org.springframework
- spring-context
- ${spring-context.version}
-
-
-
- io.codearte.jfairy
- jfairy
- 0.5.9
-
-
-
- org.apache.avro
- avro
- ${avro.version}
-
-
-
- ${project.groupId}
- pulsar-functions-local-runner-original
- ${project.version}
- test
-
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
- ${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
- spotbugs
- verify
-
- check
-
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-io
+ 2.10.5.3
+
+ pulsar-io-batch-data-generator
+ Pulsar IO :: Batch Data Generator
+
+
+ ${project.groupId}
+ pulsar-io-core
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-io-batch-discovery-triggerers
+ ${project.version}
+
+
+ org.springframework
+ spring-context
+ ${spring-context.version}
+
+
+ io.codearte.jfairy
+ jfairy
+ 0.5.9
+
+
+ org.apache.avro
+ avro
+ ${avro.version}
+
+
+ ${project.groupId}
+ pulsar-functions-local-runner-original
+ ${project.version}
+ test
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+ ${basedir}/src/main/resources/findbugsExclude.xml
+
+
+
+ spotbugs
+ verify
+
+ check
+
+
+
+
+
+
diff --git a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-batch-discovery-triggerers
Pulsar IO :: Batch Discovery Triggerers
-
${project.groupId}
pulsar-io-core
${project.version}
-
com.cronutils
cron-utils
${cron-utils.version}
-
org.springframework
spring-context
${spring-context.version}
-
-
@@ -73,5 +66,4 @@
-
diff --git a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
- org.apache.pulsar
- pulsar-io
- 2.10.5-SNAPSHOT
-
- 4.0.0
-
- pulsar-io-canal
- Pulsar IO :: Canal
-
-
- 5.3.19
- 1.1.5
-
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
- com.fasterxml.jackson.dataformat
- jackson-dataformat-yaml
-
-
- com.alibaba
- fastjson
- 1.2.83
-
-
-
- org.springframework
- spring-core
- ${spring.version}
-
-
- org.springframework
- spring-aop
- ${spring.version}
-
-
- org.springframework
- spring-context
- ${spring.version}
-
-
- org.springframework
- spring-jdbc
- ${spring.version}
-
-
- org.springframework
- spring-orm
- ${spring.version}
-
-
-
- com.alibaba.otter
- canal.protocol
- ${canal.version}
-
-
- com.alibaba.otter
- canal.client
- ${canal.version}
-
-
- ch.qos.logback
- *
-
-
- org.springframework
- *
-
-
-
-
-
- org.apache.logging.log4j
- log4j-core
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
-
-
-
\ No newline at end of file
+
+
+ io.streamnative
+ pulsar-io
+ 2.10.5.3
+
+ 4.0.0
+ pulsar-io-canal
+ Pulsar IO :: Canal
+
+ 5.3.19
+ 1.1.5
+
+
+
+ ${project.groupId}
+ pulsar-io-core
+ ${project.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+
+
+ com.alibaba
+ fastjson
+ 1.2.83
+
+
+ org.springframework
+ spring-core
+ ${spring.version}
+
+
+ org.springframework
+ spring-aop
+ ${spring.version}
+
+
+ org.springframework
+ spring-context
+ ${spring.version}
+
+
+ org.springframework
+ spring-jdbc
+ ${spring.version}
+
+
+ org.springframework
+ spring-orm
+ ${spring.version}
+
+
+ com.alibaba.otter
+ canal.protocol
+ ${canal.version}
+
+
+ com.alibaba.otter
+ canal.client
+ ${canal.version}
+
+
+ ch.qos.logback
+ *
+
+
+ org.springframework
+ *
+
+
+
+
+ org.apache.logging.log4j
+ log4j-core
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+
+
diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml
index 901f752771cce..556bc8d9f18e5 100644
--- a/pulsar-io/cassandra/pom.xml
+++ b/pulsar-io/cassandra/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-cassandra
Pulsar IO :: Cassandra
-
-
${project.groupId}
pulsar-io-core
${project.version}
-
com.fasterxml.jackson.core
jackson-databind
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.datastax.cassandra
cassandra-driver-core
${cassandra-driver-core.version}
-
-
diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml
index 1517ad01d3df8..e8845067c04d6 100644
--- a/pulsar-io/common/pom.xml
+++ b/pulsar-io/common/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
-
- org.apache.pulsar
- pulsar-io
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-common
- Pulsar IO :: IO Common
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
- ${project.groupId}
- pulsar-common
- ${project.version}
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
-
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
- ${basedir}/src/main/resources/findbugsExclude.xml
-
-
-
- spotbugs
- verify
-
- check
-
-
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-io
+ 2.10.5.3
+
+ pulsar-io-common
+ Pulsar IO :: IO Common
+
+
+ ${project.groupId}
+ pulsar-io-core
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-common
+ ${project.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+
+
+ com.github.spotbugs
+ spotbugs-maven-plugin
+ ${spotbugs-maven-plugin.version}
+
+ ${basedir}/src/main/resources/findbugsExclude.xml
+
+
+
+ spotbugs
+ verify
+
+ check
+
+
+
+
+
+
diff --git a/pulsar-io/common/src/main/resources/findbugsExclude.xml b/pulsar-io/common/src/main/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-io/common/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/common/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-core
Pulsar IO :: IO
-
${project.groupId}
@@ -37,7 +35,6 @@
${project.version}
-
@@ -59,5 +56,4 @@
-
diff --git a/pulsar-io/core/src/main/resources/findbugsExclude.xml b/pulsar-io/core/src/main/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-io/core/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/core/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- org.apache.pulsar
- pulsar-io
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-data-generator
- Pulsar IO :: Data Generator
-
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
-
- ${project.groupId}
- pulsar-config-validation
- ${project.version}
-
-
-
- io.codearte.jfairy
- jfairy
- 0.5.9
-
-
-
- org.apache.avro
- avro
- ${avro.version}
-
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-io
+ 2.10.5.3
+
+ pulsar-io-data-generator
+ Pulsar IO :: Data Generator
+
+
+ ${project.groupId}
+ pulsar-io-core
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-config-validation
+ ${project.version}
+
+
+ io.codearte.jfairy
+ jfairy
+ 0.5.9
+
+
+ org.apache.avro
+ avro
+ ${avro.version}
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+
diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml
index c68c37a607b3c..13af929d5cb84 100644
--- a/pulsar-io/debezium/core/pom.xml
+++ b/pulsar-io/debezium/core/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io-debezium
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium-core
Pulsar IO :: Debezium :: Core
-
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-client-original
${project.version}
-
io.debezium
debezium-core
${debezium.version}
-
${project.groupId}
pulsar-io-kafka-connect-adaptor
${project.version}
-
org.apache.kafka
kafka_${scala.binary.version}
${kafka-client.version}
-
org.apache.kafka
connect-runtime
@@ -73,21 +65,18 @@
-
${project.groupId}
pulsar-broker
${project.version}
test
-
${project.groupId}
testmocks
${project.version}
test
-
${project.groupId}
pulsar-broker
@@ -95,14 +84,11 @@
test
test-jar
-
io.debezium
debezium-connector-mysql
${debezium.version}
test
-
-
diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml
index 76170a06ba380..277ed6732cfeb 100644
--- a/pulsar-io/debezium/mongodb/pom.xml
+++ b/pulsar-io/debezium/mongodb/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- org.apache.pulsar
- pulsar-io-debezium
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-debezium-mongodb
- Pulsar IO :: Debezium :: mongodb
-
-
-
-
- ${project.groupId}
- pulsar-io-debezium-core
- ${project.version}
-
-
-
-
- org.mongodb
- mongodb-driver-sync
- 4.2.2
-
-
-
- io.debezium
- debezium-connector-mongodb
- ${debezium.version}
-
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ pulsar-io-debezium
+ 2.10.5.3
+
+ pulsar-io-debezium-mongodb
+ Pulsar IO :: Debezium :: mongodb
+
+
+ ${project.groupId}
+ pulsar-io-debezium-core
+ ${project.version}
+
+
+
+ org.mongodb
+ mongodb-driver-sync
+ 4.2.2
+
+
+ io.debezium
+ debezium-connector-mongodb
+ ${debezium.version}
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+
diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml
index 3a722020cabf0..8ad1b65449615 100644
--- a/pulsar-io/debezium/mssql/pom.xml
+++ b/pulsar-io/debezium/mssql/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io-debezium
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium-mssql
Pulsar IO :: Debezium :: Microsoft SQL
-
-
${project.groupId}
pulsar-io-debezium-core
${project.version}
-
io.debezium
debezium-connector-sqlserver
${debezium.version}
-
-
@@ -54,5 +48,4 @@
-
diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml
index a5bd266612740..21909e1d3ebb1 100644
--- a/pulsar-io/debezium/mysql/pom.xml
+++ b/pulsar-io/debezium/mysql/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io-debezium
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium-mysql
Pulsar IO :: Debezium :: mysql
-
-
${project.groupId}
pulsar-io-debezium-core
${project.version}
-
io.debezium
debezium-connector-mysql
${debezium.version}
-
-
@@ -55,8 +49,6 @@
-
-
@@ -65,5 +57,4 @@
-
diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml
index 75cdf971c414e..71a75687eddd0 100644
--- a/pulsar-io/debezium/oracle/pom.xml
+++ b/pulsar-io/debezium/oracle/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io-debezium
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium-oracle
Pulsar IO :: Debezium :: oracle
-
-
${project.groupId}
pulsar-io-debezium-core
${project.version}
-
io.debezium
debezium-connector-oracle
${debezium.version}
-
-
@@ -54,5 +48,4 @@
-
diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml
index 53890b23fb617..6c0b2e5be1a0d 100644
--- a/pulsar-io/debezium/pom.xml
+++ b/pulsar-io/debezium/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
pom
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium
Pulsar IO :: Debezium
-
@@ -70,7 +68,6 @@
-
core
mysql
@@ -79,5 +76,4 @@
oracle
mssql
-
diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml
index bb91815add537..b49c9fba982a2 100644
--- a/pulsar-io/debezium/postgres/pom.xml
+++ b/pulsar-io/debezium/postgres/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io-debezium
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-debezium-postgres
Pulsar IO :: Debezium :: postgres
-
-
${project.groupId}
pulsar-io-debezium-core
${project.version}
-
io.debezium
debezium-connector-postgres
${debezium.version}
-
org.postgresql
postgresql
${debezium.postgresql.version}
runtime
-
-
@@ -61,5 +54,4 @@
-
diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml
index 64ff9e833d0e5..6273e723c9350 100644
--- a/pulsar-io/docs/pom.xml
+++ b/pulsar-io/docs/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-docs
Pulsar IO :: Docs
-
-
${project.groupId}
pulsar-io-core
@@ -45,7 +42,6 @@
com.beust
jcommander
-
${project.groupId}
@@ -193,7 +189,6 @@
${project.version}
-
@@ -242,5 +237,4 @@
-
diff --git a/pulsar-io/docs/src/main/resources/findbugsExclude.xml b/pulsar-io/docs/src/main/resources/findbugsExclude.xml
index 07f4609cfff29..47c3d73af5f06 100644
--- a/pulsar-io/docs/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/docs/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-dynamodb
Pulsar IO :: DynamoDB
-
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-functions-instance
${project.version}
provided
-
${project.groupId}
pulsar-io-aws
${project.version}
-
org.apache.commons
commons-lang3
-
com.fasterxml.jackson.core
jackson-databind
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.fasterxml.jackson.dataformat
jackson-dataformat-cbor
-
com.google.code.gson
gson
-
com.amazonaws
aws-java-sdk-core
-
com.amazonaws
@@ -90,9 +78,7 @@
1.5.1
-
-
@@ -101,5 +87,4 @@
-
diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml
index 8dc03450361ea..c25dbcbc7c27f 100644
--- a/pulsar-io/elastic-search/pom.xml
+++ b/pulsar-io/elastic-search/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- org.apache.pulsar
- pulsar-io
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-flume
- Pulsar IO :: Flume
-
-
- 1.8.2
-
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
- com.fasterxml.jackson.dataformat
- jackson-dataformat-yaml
-
-
-
- org.apache.flume
- flume-ng-node
- 1.9.0
- pom
-
-
- org.apache.avro
- avro-ipc
-
-
- avro
- org.apache.avro
-
-
-
-
- org.apache.avro
- avro
- ${avro.version}
-
-
- commons-lang
- commons-lang
- 2.6
-
-
- org.apache.avro
- avro-ipc
- ${avro.version}
-
-
- org.mortbay.jetty
- servlet-api
-
-
- io.netty
- netty
-
-
-
-
- org.apache.curator
- curator-framework
- ${curator.version}
-
-
- org.apache.curator
- curator-test
- ${curator.version}
- test
-
-
-
- io.dropwizard.metrics
- metrics-core
- test
-
-
-
- org.xerial.snappy
- snappy-java
- test
-
-
-
- com.google.guava
- guava
- ${guava.version}
-
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
-
-
+
+ io.dropwizard.metrics
+ metrics-core
+ test
+
+
+
+ org.xerial.snappy
+ snappy-java
+ test
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+
+
+
-
- owasp-dependency-check
-
-
-
- org.owasp
- dependency-check-maven
- ${dependency-check-maven.version}
-
-
-
- aggregate
-
- none
-
-
-
-
-
-
-
-
+
+ owasp-dependency-check
+
+
+
+ org.owasp
+ dependency-check-maven
+ ${dependency-check-maven.version}
+
+
+
+ aggregate
+
+ none
+
+
+
+
+
+
+
diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml
index a333ddebd2006..b6eade11fc55d 100644
--- a/pulsar-io/hbase/pom.xml
+++ b/pulsar-io/hbase/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- pulsar-io
- org.apache.pulsar
- 2.10.5-SNAPSHOT
-
- pulsar-io-hbase
- Pulsar IO :: Hbase
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
-
- ${project.groupId}
- pulsar-functions-instance
- ${project.version}
-
-
-
- ${project.groupId}
- pulsar-client-original
- ${project.version}
-
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
- com.fasterxml.jackson.dataformat
- jackson-dataformat-yaml
-
-
-
- com.google.guava
- guava
-
-
-
- org.apache.hbase
- hbase-client
- ${hbase.version}
-
-
- log4j
- log4j
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
-
- org.apache.hbase
- hbase-common
- ${hbase.version}
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
-
-
-
- owasp-dependency-check
-
-
-
- org.owasp
- dependency-check-maven
- ${dependency-check-maven.version}
-
-
-
- aggregate
-
- none
-
-
-
-
-
-
-
-
+
+ owasp-dependency-check
+
+
+
+ org.owasp
+ dependency-check-maven
+ ${dependency-check-maven.version}
+
+
+
+ aggregate
+
+ none
+
+
+
+
+
+
+
diff --git a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml
index c2b520a765643..460cbd497c15f 100644
--- a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml
+++ b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml
@@ -1,3 +1,4 @@
+
-
- hbase.cluster.distributed
- true
-
-
- hbase.rootdir
- hdfs://localhost:8020/hbase
-
-
- hbase.zookeeper.quorum
- localhost
-
-
- hbase.zookeeper.property.clientPort
- 2181
-
-
- zookeeper.znode.parent
- /hbase
-
+
+ hbase.cluster.distributed
+ true
+
+
+ hbase.rootdir
+ hdfs://localhost:8020/hbase
+
+
+ hbase.zookeeper.quorum
+ localhost
+
+
+ hbase.zookeeper.property.clientPort
+ 2181
+
+
+ zookeeper.znode.parent
+ /hbase
+
diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml
index 3843bbd7c250b..7c45849569072 100644
--- a/pulsar-io/hdfs2/pom.xml
+++ b/pulsar-io/hdfs2/pom.xml
@@ -1,3 +1,4 @@
+
-
- owasp-dependency-check
-
-
-
- org.owasp
- dependency-check-maven
- ${dependency-check-maven.version}
-
-
-
- aggregate
-
- none
-
-
-
-
-
-
-
-
-
\ No newline at end of file
+
+ owasp-dependency-check
+
+
+
+ org.owasp
+ dependency-check-maven
+ ${dependency-check-maven.version}
+
+
+
+ aggregate
+
+ none
+
+
+
+
+
+
+
+
diff --git a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml
index ddde8120ba518..47c3d73af5f06 100644
--- a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml
+++ b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml
index 31d1e98c47558..b5cf3c90fbe40 100644
--- a/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml
+++ b/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml
@@ -1,3 +1,4 @@
+
-
- fs.defaultFS
- hdfs://0.0.0.0:8020
-
-
- io.compression.codecs
- org.apache.hadoop.io.compress.GzipCodec,
+
+ fs.defaultFS
+ hdfs://0.0.0.0:8020
+
+
+ io.compression.codecs
+ org.apache.hadoop.io.compress.GzipCodec,
org.apache.hadoop.io.compress.DefaultCodec,
org.apache.hadoop.io.compress.SnappyCodec
-
+
diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml
index bb722f1f63470..1b35ebd585012 100644
--- a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml
+++ b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml
@@ -1,3 +1,4 @@
+
-
- dfs.replication
- 1
-
-
- dfs.client.use.datanode.hostname
- true
-
-
- dfs.support.append
- true
-
+
+ dfs.replication
+ 1
+
+
+ dfs.client.use.datanode.hostname
+ true
+
+
+ dfs.support.append
+ true
+
diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml
index 5d14934b49305..eac43e4bf1470 100644
--- a/pulsar-io/hdfs3/pom.xml
+++ b/pulsar-io/hdfs3/pom.xml
@@ -1,3 +1,4 @@
+
-
\ No newline at end of file
+
diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml
index 31d1e98c47558..b5cf3c90fbe40 100644
--- a/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml
+++ b/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml
@@ -1,3 +1,4 @@
+
-
- fs.defaultFS
- hdfs://0.0.0.0:8020
-
-
- io.compression.codecs
- org.apache.hadoop.io.compress.GzipCodec,
+
+ fs.defaultFS
+ hdfs://0.0.0.0:8020
+
+
+ io.compression.codecs
+ org.apache.hadoop.io.compress.GzipCodec,
org.apache.hadoop.io.compress.DefaultCodec,
org.apache.hadoop.io.compress.SnappyCodec
-
+
diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml
index bb722f1f63470..1b35ebd585012 100644
--- a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml
+++ b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml
@@ -1,3 +1,4 @@
+
-
- dfs.replication
- 1
-
-
- dfs.client.use.datanode.hostname
- true
-
-
- dfs.support.append
- true
-
+
+ dfs.replication
+ 1
+
+
+ dfs.client.use.datanode.hostname
+ true
+
+
+ dfs.support.append
+ true
+
diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml
index 3e7420a364ce0..ff60bbe313d79 100644
--- a/pulsar-io/influxdb/pom.xml
+++ b/pulsar-io/influxdb/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
- pulsar-io
- org.apache.pulsar
- 2.10.5-SNAPSHOT
-
-
- pulsar-io-influxdb
- Pulsar IO :: InfluxDB
-
-
-
- ${project.groupId}
- pulsar-io-core
- ${project.version}
-
-
- ${project.groupId}
- pulsar-functions-instance
- ${project.version}
-
-
- ${project.groupId}
- pulsar-client-original
- ${project.version}
-
-
-
- com.influxdb
- influxdb-client-java
- 4.0.0
-
-
-
- org.influxdb
- influxdb-java
- 2.22
-
-
- com.squareup.okhttp3
- *
-
-
-
-
-
- org.apache.commons
- commons-lang3
- 3.4
-
-
- commons-collections
- commons-collections
- 3.2.2
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
-
-
-
\ No newline at end of file
+
+ 4.0.0
+
+ pulsar-io
+ io.streamnative
+ 2.10.5.3
+
+ pulsar-io-influxdb
+ Pulsar IO :: InfluxDB
+
+
+ ${project.groupId}
+ pulsar-io-core
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-functions-instance
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-client-original
+ ${project.version}
+
+
+ com.influxdb
+ influxdb-client-java
+ 4.0.0
+
+
+ org.influxdb
+ influxdb-java
+ 2.22
+
+
+ com.squareup.okhttp3
+ *
+
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.4
+
+
+ commons-collections
+ commons-collections
+ 3.2.2
+
+
+
+
+
+ org.apache.nifi
+ nifi-nar-maven-plugin
+
+
+
+
diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml
index f3b0abb1b4f6b..66897e888cba9 100644
--- a/pulsar-io/jdbc/clickhouse/pom.xml
+++ b/pulsar-io/jdbc/clickhouse/pom.xml
@@ -1,3 +1,4 @@
+
-
+
pulsar-io-jdbc
- org.apache.pulsar
- 2.10.5-SNAPSHOT
+ io.streamnative
+ 2.10.5.3
4.0.0
-
pulsar-io-jdbc-clickhouse
Pulsar IO :: Jdbc :: ClickHouse
-
${project.groupId}
@@ -44,7 +41,6 @@
runtime
-
diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml
index 944b2962ab68a..248407aa47bb6 100644
--- a/pulsar-io/jdbc/core/pom.xml
+++ b/pulsar-io/jdbc/core/pom.xml
@@ -1,3 +1,4 @@
+
-
+
pulsar-io-jdbc
- org.apache.pulsar
- 2.10.5-SNAPSHOT
+ io.streamnative
+ 2.10.5.3
4.0.0
-
pulsar-io-jdbc-core
Pulsar IO :: Jdbc :: Core
-
${project.groupId}
pulsar-io-core
${project.version}
-
org.apache.avro
avro
${avro.version}
-
com.fasterxml.jackson.core
jackson-databind
-
com.google.guava
guava
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
-
\ No newline at end of file
+
diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml
index c7127fea55183..295b05cdb1d81 100644
--- a/pulsar-io/jdbc/mariadb/pom.xml
+++ b/pulsar-io/jdbc/mariadb/pom.xml
@@ -1,3 +1,4 @@
+
-
+
pulsar-io-jdbc
- org.apache.pulsar
- 2.10.5-SNAPSHOT
+ io.streamnative
+ 2.10.5.3
4.0.0
-
pulsar-io-jdbc-mariadb
Pulsar IO :: Jdbc :: Mariadb
-
${project.groupId}
@@ -52,4 +49,4 @@
-
\ No newline at end of file
+
diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml
index f9f774f327f48..d97bbc86a7580 100644
--- a/pulsar-io/jdbc/pom.xml
+++ b/pulsar-io/jdbc/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
pom
@@ -30,12 +30,10 @@
sqlite
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-jdbc
Pulsar IO :: Jdbc
-
diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml
index 0fbcdc2c25b06..62e25338f6fe1 100644
--- a/pulsar-io/jdbc/postgres/pom.xml
+++ b/pulsar-io/jdbc/postgres/pom.xml
@@ -1,3 +1,4 @@
+
-
+
pulsar-io-jdbc
- org.apache.pulsar
- 2.10.5-SNAPSHOT
+ io.streamnative
+ 2.10.5.3
4.0.0
-
pulsar-io-jdbc-postgres
Pulsar IO :: Jdbc :: Postgres
-
${project.groupId}
@@ -44,7 +41,6 @@
runtime
-
diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml
index 257e5c73eb304..b62ee375bc7f6 100644
--- a/pulsar-io/jdbc/sqlite/pom.xml
+++ b/pulsar-io/jdbc/sqlite/pom.xml
@@ -1,3 +1,4 @@
+
-
+
pulsar-io-jdbc
- org.apache.pulsar
- 2.10.5-SNAPSHOT
+ io.streamnative
+ 2.10.5.3
4.0.0
pulsar-io-jdbc-sqlite
Pulsar IO :: Jdbc :: Sqlite
-
${project.groupId}
@@ -55,7 +53,6 @@
test
-
@@ -64,4 +61,4 @@
-
\ No newline at end of file
+
diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml
index a14a5d899b104..490cefdf358a3 100644
--- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml
+++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-kafka-connect-adaptor-nar
Pulsar IO :: Kafka Connect Adaptor NAR
-
${project.groupId}
@@ -37,8 +35,6 @@
${project.version}
-
-
@@ -48,7 +44,30 @@
pulsar-io-kafka-connect-adaptor-${project.version}
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+ 3.2.0
+
+
+ default-jar
+ package
+
+ jar
+
+
+
+ javadoc-jar
+ package
+
+ jar
+
+
+ javadoc
+
+
+
+
-
diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml
index bc7b9a18c7a5d..57a59106f6106 100644
--- a/pulsar-io/kafka-connect-adaptor/pom.xml
+++ b/pulsar-io/kafka-connect-adaptor/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-kafka-connect-adaptor
Pulsar IO :: Kafka Connect Adaptor
-
-
${project.groupId}
pulsar-io-core
${project.version}
-
${project.groupId}
pulsar-io-common
${project.version}
-
org.apache.kafka
kafka_${scala.binary.version}
${kafka-client.version}
-
org.apache.kafka
connect-runtime
@@ -61,25 +55,21 @@
-
org.apache.kafka
connect-json
${kafka-client.version}
-
org.apache.kafka
connect-api
${kafka-client.version}
-
${project.groupId}
pulsar-client-original
${project.version}
-
${project.groupId}
kafka-connect-avro-converter-shaded
@@ -95,28 +85,24 @@
-
${project.groupId}
pulsar-broker
${project.version}
test
-
org.apache.kafka
connect-file
${kafka-client.version}
test
-
${project.groupId}
testmocks
${project.version}
test
-
${project.groupId}
pulsar-broker
@@ -124,12 +110,15 @@
test
test-jar
-
com.google.protobuf
protobuf-java
provided
-
+
+ com.google.protobuf
+ protobuf-java
+ 3.5.1
+
diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml
index 863e780d7e44c..af4c805b79244 100644
--- a/pulsar-io/kafka/pom.xml
+++ b/pulsar-io/kafka/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
- org.apache.pulsar
+ io.streamnative
pulsar-io
- 2.10.5-SNAPSHOT
+ 2.10.5.3
-
pulsar-io-kafka
Pulsar IO :: Kafka
-
@@ -44,63 +42,51 @@
-
-
${project.groupId}
pulsar-io-core
${project.version}
provided
-
${project.groupId}
pulsar-client-original
${project.version}
-
com.fasterxml.jackson.core
jackson-databind
-
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-
com.google.guava
guava
-
org.apache.kafka
kafka-clients
${kafka-client.version}
-
io.confluent
kafka-schema-registry-client
${kafka.confluent.schemaregistryclient.version}
-
io.confluent
kafka-avro-serializer
${kafka.confluent.avroserializer.version}
-
org.awaitility
awaitility
test
-
-
diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
index 3933375512baf..2c4c3dd3de5a7 100644
--- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
+++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java
@@ -26,7 +26,6 @@
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@@ -43,7 +42,6 @@
import org.apache.pulsar.common.schema.KeyValueEncodingType;
import org.apache.pulsar.functions.api.KVRecord;
import org.apache.pulsar.functions.api.Record;
-import org.apache.pulsar.io.core.PushSource;
import org.apache.pulsar.io.core.SourceContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +49,7 @@
/**
* Simple Kafka Source to transfer messages from a Kafka topic.
*/
-public abstract class KafkaAbstractSource extends PushSource {
+public abstract class KafkaAbstractSource extends KafkaPushSource {
private static final Logger LOG = LoggerFactory.getLogger(KafkaAbstractSource.class);
@@ -126,7 +124,6 @@ public void open(Map config, SourceContext sourceContext) throws
throw new IllegalArgumentException("Unable to instantiate Kafka consumer", ex);
}
this.start();
- running = true;
}
protected Properties beforeCreateConsumer(Properties props) {
@@ -151,47 +148,36 @@ public void close() throws InterruptedException {
@SuppressWarnings("unchecked")
public void start() {
+ LOG.info("Starting subscribe kafka source on {}", kafkaSourceConfig.getTopic());
+ consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic()));
runnerThread = new Thread(() -> {
- LOG.info("Starting kafka source on {}", kafkaSourceConfig.getTopic());
- consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic()));
LOG.info("Kafka source started.");
while (running) {
- ConsumerRecords
+
+
+
+
diff --git a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml
index e32e9b464a15a..7c8bf2d74c577 100644
--- a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml
+++ b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml
@@ -1,3 +1,4 @@
+
-
+
-
-
-
-
-
-
+
+
+
+
+
+
diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml
index 03c2f4436e35c..59f118e2c1945 100644
--- a/tests/pulsar-client-all-shade-test/pom.xml
+++ b/tests/pulsar-client-all-shade-test/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
- org.apache.pulsar.tests
- tests-parent
- 2.10.5-SNAPSHOT
-
-
- pulsar-client-all-shade-test
- jar
- Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test
-
-
-
-
- org.apache.pulsar
- pulsar-client-all
- ${project.version}
- test
-
-
-
- org.testcontainers
- testcontainers
- test
-
-
-
- org.apache.pulsar
- bouncy-castle-bc
- ${project.version}
- pkg
-
-
-
- org.apache.pulsar
- pulsar-client-messagecrypto-bc
- ${project.version}
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative.tests
+ tests-parent
+ 2.10.5.3
+
+ pulsar-client-all-shade-test
+ jar
+ Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test
+
+
+ io.streamnative
+ pulsar-client-all
+ ${project.version}
+ test
+
+
+ org.testcontainers
+ testcontainers
+ test
+
+
+ io.streamnative
+ bouncy-castle-bc
+ ${project.version}
+ pkg
+
+
+ io.streamnative
+ pulsar-client-messagecrypto-bc
+ ${project.version}
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ test-jar
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ true
+
+ ${project.version}
+ ${project.build.directory}
+
+
+
+
+
+
+
+ ShadeTests
+
+
+ ShadeTests
+
+
+
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- test-jar
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
-
-
- true
-
- ${project.version}
- ${project.build.directory}
-
-
-
-
-
-
-
-
- ShadeTests
-
-
- ShadeTests
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
-
- ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+ ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G
-Dio.netty.leakDetectionLevel=advanced
${test.additional.args}
- false
-
- src/test/resources/pulsar.xml
-
- 1
-
-
-
-
-
-
+ false
+
+ src/test/resources/pulsar.xml
+
+ 1
+
+
+
+
+
+
diff --git a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml
index 746cc3d1b9e33..c4fc288600bae 100644
--- a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml
+++ b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml
@@ -1,3 +1,4 @@
+
-
+
-
-
-
-
-
-
+
+
+
+
+
+
diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml
index af22ca795e109..75072cab7df4e 100644
--- a/tests/pulsar-client-shade-test/pom.xml
+++ b/tests/pulsar-client-shade-test/pom.xml
@@ -1,4 +1,4 @@
-
+
-
- 4.0.0
-
-
- org.apache.pulsar.tests
- tests-parent
- 2.10.5-SNAPSHOT
-
-
- pulsar-client-shade-test
- jar
- Apache Pulsar :: Tests :: Pulsar-Client-Shade Test
-
-
-
-
- org.apache.pulsar
- pulsar-client
- ${project.version}
- test
-
-
-
- org.apache.pulsar
- pulsar-client-admin
- ${project.version}
- test
-
-
-
- org.testcontainers
- testcontainers
- test
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative.tests
+ tests-parent
+ 2.10.5.3
+
+ pulsar-client-shade-test
+ jar
+ Apache Pulsar :: Tests :: Pulsar-Client-Shade Test
+
+
+ io.streamnative
+ pulsar-client
+ ${project.version}
+ test
+
+
+ io.streamnative
+ pulsar-client-admin
+ ${project.version}
+ test
+
+
+ org.testcontainers
+ testcontainers
+ test
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ test-jar
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ true
+
+ ${project.version}
+ ${project.build.directory}
+
+
+
+
+
+
+
+ ShadeTests
+
+
+ ShadeTests
+
+
+
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- test-jar
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
-
-
- true
-
- ${project.version}
- ${project.build.directory}
-
-
-
-
-
-
-
-
- ShadeTests
-
-
- ShadeTests
-
-
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
-
- ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+ ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G
-Dio.netty.leakDetectionLevel=advanced
${test.additional.args}
- false
-
- src/test/resources/pulsar.xml
-
- 1
-
-
-
-
-
-
+ false
+
+ src/test/resources/pulsar.xml
+
+ 1
+
+
+
+
+
+
diff --git a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml
index df07642a01851..b83f6d760bb22 100644
--- a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml
+++ b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml
@@ -1,3 +1,4 @@
+
-
+
-
-
-
-
-
-
+
+
+
+
+
+
diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml
index 9739a04ece167..90a74d72594e2 100644
--- a/tiered-storage/file-system/pom.xml
+++ b/tiered-storage/file-system/pom.xml
@@ -1,3 +1,4 @@
+
-
- 4.0.0
-
-
- org.apache.pulsar
- tiered-storage-parent
- 2.10.5-SNAPSHOT
- ..
-
-
- tiered-storage-file-system
- Apache Pulsar :: Tiered Storage :: File System
-
-
- ${project.groupId}
- managed-ledger
- ${project.version}
-
-
- org.apache.hadoop
- hadoop-common
- ${hdfs-offload-version3}
-
-
- log4j
- log4j
-
-
- org.slf4j
- slf4j-log4j12
-
-
-
-
-
- org.apache.hadoop
- hadoop-hdfs-client
- ${hdfs-offload-version3}
-
-
- org.apache.avro
- avro
-
-
- org.mortbay.jetty
- jetty
-
-
- com.sun.jersey
- jersey-core
-
-
- com.sun.jersey
- jersey-server
-
-
- javax.servlet
- servlet-api
-
-
-
-
-
- org.apache.avro
- avro
- ${avro.version}
-
-
-
- net.minidev
- json-smart
- ${json-smart.version}
-
-
- com.google.protobuf
- protobuf-java
-
-
-
- ${project.groupId}
- testmocks
- ${project.version}
- test
-
-
-
- org.apache.hadoop
- hadoop-minicluster
- ${hdfs-offload-version3}
- test
-
-
- io.netty
- netty-all
-
-
-
-
-
+
+ 4.0.0
+
+ io.streamnative
+ tiered-storage-parent
+ 2.10.5.3
+ ..
+
+ tiered-storage-file-system
+ Apache Pulsar :: Tiered Storage :: File System
+
+
+ ${project.groupId}
+ managed-ledger
+ ${project.version}
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hdfs-offload-version3}
+
+
+ log4j
+ log4j
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs-client
+ ${hdfs-offload-version3}
+
+
+ org.apache.avro
+ avro
+
+
+ org.mortbay.jetty
+ jetty
+
+
+ com.sun.jersey
+ jersey-core
+
+
+ com.sun.jersey
+ jersey-server
+
+
+ javax.servlet
+ servlet-api
+
+
+
+
+
+ org.apache.avro
+ avro
+ ${avro.version}
+
+
+
+ net.minidev
+ json-smart
+ ${json-smart.version}
+
+
+ com.google.protobuf
+ protobuf-java
+
+
+ ${project.groupId}
+ testmocks
+ ${project.version}
+ test
+
+
+ org.apache.hadoop
+ hadoop-minicluster
+ ${hdfs-offload-version3}
+ test
+
+
io.netty
- netty-codec-http
-
-
-
- org.eclipse.jetty
- jetty-server
- test
-
-
- org.eclipse.jetty
- jetty-alpn-conscrypt-server
- test
-
-
- org.eclipse.jetty
- jetty-servlet
- test
-
-
- org.eclipse.jetty
- jetty-util
- test
-
-
-
-
-
-
- org.apache.nifi
- nifi-nar-maven-plugin
-
-
- com.github.spotbugs
- spotbugs-maven-plugin
- ${spotbugs-maven-plugin.version}
-
- ${basedir}/../src/main/resources/findbugsExclude.xml
-
-
-
- spotbugs
- verify
-
- check
-
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
-
- checkstyle
- verify
-
- check
-
-
-
-
-
-
-
-
-
- owasp-dependency-check
-
-
-
- org.owasp
- dependency-check-maven
- ${dependency-check-maven.version}
-
-
-
- aggregate
-
- none
-
-
-
-
-
-
-
+
+ owasp-dependency-check
+
+
+
+ org.owasp
+ dependency-check-maven
+ ${dependency-check-maven.version}
+
+
+
+ aggregate
+
+ none
+
+
+
+
+
+
+
diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml
index 65e777fcf615b..a182ffa608400 100644
--- a/tiered-storage/jcloud/pom.xml
+++ b/tiered-storage/jcloud/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
-
- org.apache.pulsar
+ io.streamnative
tiered-storage-parent
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
tiered-storage-jcloud
Apache Pulsar :: Tiered Storage :: JCloud
-
${project.groupId}
managed-ledger
${project.version}
-
${project.groupId}
jclouds-shaded
@@ -70,7 +66,6 @@
-
org.apache.jclouds
jclouds-allblobstore
@@ -81,12 +76,10 @@
com.amazonaws
aws-java-sdk-core
-
com.amazonaws
aws-java-sdk-sts
-
${project.groupId}
testmocks
@@ -99,7 +92,6 @@
${jclouds.version}
provided
-
javax.xml.bind
jaxb-api
@@ -111,13 +103,11 @@
runtime
-
com.sun.activation
javax.activation
runtime
-
@@ -125,7 +115,6 @@
org.apache.nifi
nifi-nar-maven-plugin
-
com.github.spotbugs
spotbugs-maven-plugin
diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml
index 3178ff483b59e..7116dbdc65321 100644
--- a/tiered-storage/pom.xml
+++ b/tiered-storage/pom.xml
@@ -1,3 +1,4 @@
+
-
+
4.0.0
pom
- org.apache.pulsar
+ io.streamnative
pulsar
- 2.10.5-SNAPSHOT
+ 2.10.5.3
..
-
tiered-storage-parent
Apache Pulsar :: Tiered Storage :: Parent
-
${project.version}
-
jcloud
file-system
-
diff --git a/tiered-storage/src/main/resources/findbugsExclude.xml b/tiered-storage/src/main/resources/findbugsExclude.xml
index 5a9926c0e8116..75c291b6e9203 100644
--- a/tiered-storage/src/main/resources/findbugsExclude.xml
+++ b/tiered-storage/src/main/resources/findbugsExclude.xml
@@ -1,3 +1,4 @@
+
-
-
-
-
+
+
+
+