diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml deleted file mode 100644 index d11f8c5e518da..0000000000000 --- a/.github/workflows/ci-cpp.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: CI - CPP, Python Tests -on: - pull_request: - branches: - - branch-* - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 - -jobs: - - cpp-tests: - name: - runs-on: ubuntu-latest - timeout-minutes: 120 - - steps: - - name: checkout - uses: actions/checkout@v2 - - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - - name: Detect changed files - id: changes - uses: apache/pulsar-test-infra/paths-filter@master - with: - filters: .github/changes-filter.yaml - - - name: Check changed files - id: check_changes - run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" - - - name: Cache local Maven repository - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/*/*/* - !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2-dependencies-core-modules- - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - with: - distribution: 'temurin' - java-version: 11 - - - name: clean disk - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - sudo apt clean - docker rmi $(docker images -q) -f - df -h - - - name: build package - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests - - - name: build cpp artifacts - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: | - echo "Build C++ client library" - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so" - pulsar-client-cpp/docker-build.sh - - - name: run c++ tests - if: ${{ steps.check_changes.outputs.docs_only != 'true' }} - run: pulsar-client-cpp/docker-tests.sh - - - name: Upload test-logs - uses: actions/upload-artifact@v2 - if: failure() - continue-on-error: true - with: - name: test-logs - path: test-logs diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml index a8b64dd182428..507936d838148 100644 --- a/bouncy-castle/bc/pom.xml +++ b/bouncy-castle/bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent - 2.10.5-SNAPSHOT + 2.10.5.3 .. - bouncy-castle-bc Apache Pulsar :: Bouncy Castle :: BC - ${project.groupId} @@ -39,23 +36,19 @@ ${project.version} provided - org.bouncycastle bcpkix-jdk15on ${bouncycastle.version} - org.bouncycastle bcprov-ext-jdk15on ${bouncycastle.version} - - de.ntcomputer @@ -73,7 +66,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml index 042572e352737..979d7fb083772 100644 --- a/bouncy-castle/bcfips-include-test/pom.xml +++ b/bouncy-castle/bcfips-include-test/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative bouncy-castle-parent - 2.10.5-SNAPSHOT + 2.10.5.3 .. - bcfips-include-test Pulsar Bouncy Castle FIPS Test Broker and client runs auth include BC FIPS verison - ${project.groupId} @@ -39,7 +37,6 @@ ${project.version} test - ${project.groupId} pulsar-broker @@ -53,7 +50,6 @@ test-jar test - ${project.groupId} pulsar-broker @@ -66,7 +62,6 @@ test - ${project.groupId} @@ -74,7 +69,6 @@ ${project.version} pkg - diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml index dffbb6c04d132..a33ebddbfea2e 100644 --- a/bouncy-castle/bcfips/pom.xml +++ b/bouncy-castle/bcfips/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative bouncy-castle-parent - 2.10.5-SNAPSHOT + 2.10.5.3 .. - bouncy-castle-bcfips Apache Pulsar :: Bouncy Castle :: BC-FIPS - ${project.groupId} @@ -39,20 +36,17 @@ ${project.version} provided - org.bouncycastle bc-fips ${bouncycastlefips.version} - org.bouncycastle bcpkix-fips ${bouncycastlefips.version} - diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml index f4b4c3c04e732..64bb5237626ca 100644 --- a/bouncy-castle/pom.xml +++ b/bouncy-castle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - bouncy-castle-parent Apache Pulsar :: Bouncy Castle :: Parent - bc bcfips - bcfips-include-test diff --git a/buildtools/pom.xml b/buildtools/pom.xml index eac579be438f7..799f416c896c0 100644 --- a/buildtools/pom.xml +++ b/buildtools/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache apache 23 - + - - org.apache.pulsar + io.streamnative buildtools - 2.10.5-SNAPSHOT + 2.10.5.3 jar Pulsar Build Tools - 1.8 1.8 @@ -50,10 +47,9 @@ 32.0.0-jre 1.10.12 2.0 - + 3.12.4 - @@ -65,9 +61,7 @@ - - org.yaml snakeyaml @@ -88,7 +82,6 @@ guice ${guice.version} - org.testng testng @@ -145,7 +138,6 @@ ${mockito.version} - @@ -240,6 +232,55 @@ **/proto/* + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + maven-gpg-plugin + + + sign-artifacts + verify + + sign + + + + @@ -264,4 +305,14 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml index 184f58487eaf0..9e12d754fe7b5 100644 --- a/buildtools/src/main/resources/log4j2.xml +++ b/buildtools/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/buildtools/src/main/resources/pulsar/checkstyle.xml b/buildtools/src/main/resources/pulsar/checkstyle.xml index bb1e86e599292..b7acd43edb8bd 100644 --- a/buildtools/src/main/resources/pulsar/checkstyle.xml +++ b/buildtools/src/main/resources/pulsar/checkstyle.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + - - - - + IMPORT CHECKS - - - - + --> + + + + - - - - - + + + + + + + - - - - + + - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + } else --> - else --> - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + - - - - - + + + diff --git a/buildtools/src/main/resources/pulsar/suppressions.xml b/buildtools/src/main/resources/pulsar/suppressions.xml index 49cfa48d3c562..78fe2ca345964 100644 --- a/buildtools/src/main/resources/pulsar/suppressions.xml +++ b/buildtools/src/main/resources/pulsar/suppressions.xml @@ -1,4 +1,4 @@ - + - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/filesystem_offload_core_site.xml b/conf/filesystem_offload_core_site.xml index d26cec2cc60f0..3c7e6d94bbe50 100644 --- a/conf/filesystem_offload_core_site.xml +++ b/conf/filesystem_offload_core_site.xml @@ -1,3 +1,4 @@ + - - - fs.defaultFS - - - - hadoop.tmp.dir - pulsar - - - io.file.buffer.size - 4096 - - - io.seqfile.compress.blocksize - 1000000 - - - io.seqfile.compression.type - BLOCK - - - io.map.index.interval - 128 - - + + + fs.defaultFS + + + + hadoop.tmp.dir + pulsar + + + io.file.buffer.size + 4096 + + + io.seqfile.compress.blocksize + 1000000 + + + io.seqfile.compression.type + BLOCK + + + io.map.index.interval + 128 + diff --git a/conf/functions_log4j2.xml b/conf/functions_log4j2.xml index 190d9be92940b..84e6ce44f51e4 100644 --- a/conf/functions_log4j2.xml +++ b/conf/functions_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - info - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + info + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + + diff --git a/conf/proxy.conf b/conf/proxy.conf index 9c63d3eda7adc..1902247e551f4 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -335,3 +335,8 @@ zookeeperSessionTimeoutMs=-1 # ZooKeeper cache expiry time in seconds # Deprecated: use metadataStoreCacheExpirySeconds zooKeeperCacheExpirySeconds=-1 + +### --- Metrics --- ### + +# Whether to enable the proxy's /metrics, /proxy-stats, and /status.html http endpoints +enableProxyStatsEndpoints=true diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml index ee43b77099062..d2427afd32736 100644 --- a/distribution/io/pom.xml +++ b/distribution/io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-io-distribution pom Pulsar :: Distribution :: IO - ${project.groupId} @@ -46,7 +43,6 @@ provided - @@ -69,9 +65,32 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - generator-connector-config @@ -151,5 +170,4 @@ - diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 1d8ea1261083f..690abc44b86e4 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -1,3 +1,4 @@ + - + bin dir @@ -38,45 +37,111 @@ . 644 - - - ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar - ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar - ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar - ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar - ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar - ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar - ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar - ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar - ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar - ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar - ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar - ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar - ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar - ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar - ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar - ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar - ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar - ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar - ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar - ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar - ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar + + ${basedir}/../../pulsar-io/cassandra/target/pulsar-io-cassandra-${project.version}.nar + + + ${basedir}/../../pulsar-io/twitter/target/pulsar-io-twitter-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka/target/pulsar-io-kafka-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/rabbitmq/target/pulsar-io-rabbitmq-${project.version}.nar + + + ${basedir}/../../pulsar-io/nsq/target/pulsar-io-nsq-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/sqlite/target/pulsar-io-jdbc-sqlite-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/mariadb/target/pulsar-io-jdbc-mariadb-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/clickhouse/target/pulsar-io-jdbc-clickhouse-${project.version}.nar + + + ${basedir}/../../pulsar-io/jdbc/postgres/target/pulsar-io-jdbc-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/batch-data-generator/target/pulsar-io-batch-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/aerospike/target/pulsar-io-aerospike-${project.version}.nar + + + ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar + + + ${basedir}/../../pulsar-io/kafka-connect-adaptor-nar/target/pulsar-io-kafka-connect-adaptor-${project.version}.nar + + + ${basedir}/../../pulsar-io/hbase/target/pulsar-io-hbase-${project.version}.nar + + + ${basedir}/../../pulsar-io/kinesis/target/pulsar-io-kinesis-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs2/target/pulsar-io-hdfs2-${project.version}.nar + + + ${basedir}/../../pulsar-io/hdfs3/target/pulsar-io-hdfs3-${project.version}.nar + + + ${basedir}/../../pulsar-io/file/target/pulsar-io-file-${project.version}.nar + + + ${basedir}/../../pulsar-io/data-generator/target/pulsar-io-data-generator-${project.version}.nar + + + ${basedir}/../../pulsar-io/canal/target/pulsar-io-canal-${project.version}.nar + + + ${basedir}/../../pulsar-io/netty/target/pulsar-io-netty-${project.version}.nar + + + ${basedir}/../../pulsar-io/mongo/target/pulsar-io-mongo-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar + + + ${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar + + + ${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar + + + ${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar + + + ${basedir}/../../pulsar-io/flume/target/pulsar-io-flume-${project.version}.nar + + + ${basedir}/../../pulsar-io/solr/target/pulsar-io-solr-${project.version}.nar + + + ${basedir}/../../pulsar-io/dynamodb/target/pulsar-io-dynamodb-${project.version}.nar + diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml index 6a0b1deb75ca6..5253f655f82ed 100644 --- a/distribution/offloaders/pom.xml +++ b/distribution/offloaders/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-offloader-distribution pom Pulsar :: Distribution :: Offloader - ${project.groupId} @@ -66,7 +63,6 @@ - @@ -90,6 +86,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/offloaders/src/assemble/offloaders.xml b/distribution/offloaders/src/assemble/offloaders.xml index 38f7eee906064..19c60f68c21d4 100644 --- a/distribution/offloaders/src/assemble/offloaders.xml +++ b/distribution/offloaders/src/assemble/offloaders.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -38,13 +37,11 @@ . 644 - ${basedir}/../../tiered-storage/jcloud/target/tiered-storage-jcloud-${project.version}.nar offloaders 644 - ${basedir}/../../tiered-storage/file-system/target/tiered-storage-file-system-${project.version}.nar offloaders diff --git a/distribution/pom.xml b/distribution/pom.xml index 5ce09b42d92e9..61e7e4bc73c40 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - distribution pom Pulsar :: Distribution - - main @@ -46,7 +42,6 @@ server - core-modules @@ -54,7 +49,6 @@ - @@ -64,6 +58,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 4a7735cee4fb3..7b05f09fe11e3 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative distribution - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-server-distribution pom Pulsar :: Distribution :: Server - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} pulsar-proxy ${project.version} - ${project.groupId} pulsar-broker-auth-sasl ${project.version} - ${project.groupId} pulsar-client-auth-sasl ${project.version} - jline jline ${jline.version} - org.apache.zookeeper zookeeper-prometheus-metrics ${zookeeper.version} - ${project.groupId} pulsar-package-bookkeeper-storage @@ -81,13 +72,11 @@ - ${project.groupId} pulsar-package-filesystem-storage ${project.version} - ${project.groupId} pulsar-client-tools @@ -99,75 +88,61 @@ - ${project.groupId} pulsar-testclient ${project.version} - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - org.apache.logging.log4j log4j-web - io.dropwizard.metrics metrics-core - io.dropwizard.metrics metrics-graphite - io.dropwizard.metrics metrics-jvm - org.xerial.snappy snappy-java - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - org.apache.logging.log4j log4j-slf4j-impl - org.apache.bookkeeper.stats prometheus-metrics-provider - io.prometheus simpleclient_log4j2 - ${project.groupId} bouncy-castle-bc ${project.version} pkg - ${project.groupId} @@ -182,7 +157,6 @@ - ${project.groupId} pulsar-functions-worker @@ -199,7 +173,6 @@ - ${project.groupId} @@ -214,7 +187,6 @@ - ${project.groupId} @@ -227,19 +199,16 @@ - io.grpc grpc-all - io.perfmark perfmark-api compile - org.apache.bookkeeper.http @@ -261,7 +230,6 @@ vertx-web - @@ -302,6 +270,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml index 12c443dc6c564..c1675577f7ef8 100644 --- a/distribution/server/src/assemble/bin.xml +++ b/distribution/server/src/assemble/bin.xml @@ -1,3 +1,4 @@ + - + bin tar.gz @@ -126,14 +125,11 @@ ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} - - org.apache.pulsar:pulsar-functions-runtime-all - + io.streamnative:pulsar-functions-runtime-all org.projectlombok:lombok - - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples *:tar.gz diff --git a/docker/pom.xml b/docker/pom.xml index 21f0a70d53116..97f2950ccced5 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 docker-images Apache Pulsar :: Docker Images @@ -39,6 +38,30 @@ ${skipBuildDistribution} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml index 1a740f710f71c..e97af3177fa67 100644 --- a/docker/pulsar-all/pom.xml +++ b/docker/pulsar-all/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images - 2.10.5-SNAPSHOT + 2.10.5.3 4.0.0 pulsar-all-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version (Include All Components) pom - ${project.groupId} @@ -64,7 +63,6 @@ - docker diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 07967210438cd..4ceeeb21ef91c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar + io.streamnative docker-images - 2.10.5-SNAPSHOT + 2.10.5.3 4.0.0 pulsar-docker-image Apache Pulsar :: Docker Images :: Pulsar Latest Version pom - false false - ${project.groupId} @@ -51,7 +49,6 @@ - docker diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml index 30c5f64e7f858..81671e211cbba 100644 --- a/jclouds-shaded/pom.xml +++ b/jclouds-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - jclouds-shaded Apache Pulsar :: Jclouds shaded - - org.apache.jclouds jclouds-allblobstore @@ -50,7 +45,6 @@ javax.annotation-api - @@ -66,7 +60,6 @@ true true false - com.google.guava:guava @@ -88,7 +81,6 @@ com.google.errorprone:* - com.google @@ -122,21 +114,44 @@ com.google.errorprone org.apache.pulsar.jcloud.shade.com.google.errorprone - - - org.apache.jclouds:jclouds-core - - - lib/gson*jar - - - + + org.apache.jclouds:jclouds-core + + + lib/gson*jar + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc diff --git a/kafka-connect-avro-converter-shaded/pom.xml b/kafka-connect-avro-converter-shaded/pom.xml index 2fa2a408031a6..90a4f25432577 100644 --- a/kafka-connect-avro-converter-shaded/pom.xml +++ b/kafka-connect-avro-converter-shaded/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - pulsar - org.apache.pulsar - 2.10.5-SNAPSHOT - .. - - - kafka-connect-avro-converter-shaded - Apache Pulsar :: Kafka Connect Avro Converter shaded - - - - - io.confluent - kafka-connect-avro-converter - ${confluent.version} - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - - - true - true - - - - io.confluent:* - io.confluent:kafka-avro-serializer - io.confluent:kafka-schema-registry-client - io.confluent:common-config - io.confluent:common-utils - org.apache.avro:* - - org.codehaus.jackson:jackson-core-asl - org.codehaus.jackson:jackson-mapper-asl - com.thoughtworks.paranamer:paranamer - org.xerial.snappy:snappy-java - org.apache.commons:commons-compress - org.tukaani:xz - - - - - io.confluent - org.apache.pulsar.kafka.shade.io.confluent - - - org.apache.avro - org.apache.pulsar.kafka.shade.avro - - - org.codehaus.jackson - org.apache.pulsar.kafka.shade.org.codehaus.jackson - - - com.thoughtworks.paranamer - org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer - - - org.xerial.snappy - org.apache.pulsar.kafka.shade.org.xerial.snappy - - - org.apache.commons - org.apache.pulsar.kafka.shade.org.apache.commons - - - org.tukaani - org.apache.pulsar.kafka.shade.org.tukaani - - - - - - - - - - - + + 4.0.0 + + pulsar + io.streamnative + 2.10.5.3 + .. + + kafka-connect-avro-converter-shaded + Apache Pulsar :: Kafka Connect Avro Converter shaded + + + + io.confluent + kafka-connect-avro-converter + ${confluent.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + + + true + true + + + io.confluent:* + io.confluent:kafka-avro-serializer + io.confluent:kafka-schema-registry-client + io.confluent:common-config + io.confluent:common-utils + org.apache.avro:* + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.thoughtworks.paranamer:paranamer + org.xerial.snappy:snappy-java + org.apache.commons:commons-compress + org.tukaani:xz + + + + + io.confluent + org.apache.pulsar.kafka.shade.io.confluent + + + org.apache.avro + org.apache.pulsar.kafka.shade.avro + + + org.codehaus.jackson + org.apache.pulsar.kafka.shade.org.codehaus.jackson + + + com.thoughtworks.paranamer + org.apache.pulsar.kafka.shade.com.thoughtworks.paranamer + + + org.xerial.snappy + org.apache.pulsar.kafka.shade.org.xerial.snappy + + + org.apache.commons + org.apache.pulsar.kafka.shade.org.apache.commons + + + org.tukaani + org.apache.pulsar.kafka.shade.org.tukaani + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 6d502d94abce9..330655f1cfd7b 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - managed-ledger Managed Ledger - org.apache.bookkeeper bookkeeper-server - org.apache.bookkeeper.stats prometheus-metrics-provider - org.apache.bookkeeper.stats codahale-metrics-provider ${bookkeeper.version} - com.google.protobuf protobuf-java - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - ${project.groupId} testmocks ${project.version} test - org.apache.zookeeper zookeeper @@ -85,29 +74,25 @@ test - io.dropwizard.metrics - metrics-core - test + io.dropwizard.metrics + metrics-core + test - org.xerial.snappy - snappy-java - test + org.xerial.snappy + snappy-java + test - org.awaitility awaitility test - org.slf4j slf4j-api - - diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java index 173a8098d952d..b5b5cd21ff80b 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java @@ -139,6 +139,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.util.DateFormatter; import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.common.util.LazyLoadableValue; import org.apache.pulsar.common.util.collections.ConcurrentLongHashMap; import org.apache.pulsar.metadata.api.Stat; import org.slf4j.Logger; @@ -2452,15 +2453,13 @@ private void maybeOffload(CompletableFuture finalPromise) { } } - private boolean hasLedgerRetentionExpired(long ledgerTimestamp) { - return config.getRetentionTimeMillis() >= 0 - && clock.millis() - ledgerTimestamp > config.getRetentionTimeMillis(); + private boolean hasLedgerRetentionExpired(long retentionTimeMs, long ledgerTimestamp) { + return retentionTimeMs >= 0 && clock.millis() - ledgerTimestamp > retentionTimeMs; } - private boolean isLedgerRetentionOverSizeQuota(long sizeToDelete) { + private boolean isLedgerRetentionOverSizeQuota(long retentionSizeInMB, long totalSizeOfML, long sizeToDelete) { // Handle the -1 size limit as "infinite" size quota - return config.getRetentionSizeInMB() >= 0 - && TOTAL_SIZE_UPDATER.get(this) - sizeToDelete >= config.getRetentionSizeInMB() * MegaByte; + return retentionSizeInMB >= 0 && totalSizeOfML - sizeToDelete >= retentionSizeInMB * MegaByte; } boolean isOffloadedNeedsDelete(OffloadContext offload, Optional offloadPolicies) { @@ -2518,6 +2517,11 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } long slowestReaderLedgerId = -1; + final LazyLoadableValue slowestNonDurationLedgerId = + new LazyLoadableValue(() -> getTheSlowestNonDurationReadPosition().getLedgerId()); + final long retentionSizeInMB = config.getRetentionSizeInMB(); + final long retentionTimeMs = config.getRetentionTimeMillis(); + final long totalSizeOfML = TOTAL_SIZE_UPDATER.get(this); if (!cursors.hasDurableCursors()) { // At this point the lastLedger will be pointing to the // ledger that has just been closed, therefore the +1 to @@ -2540,7 +2544,10 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { long totalSizeToDelete = 0; // skip ledger if retention constraint met - for (LedgerInfo ls : ledgers.headMap(slowestReaderLedgerId, false).values()) { + Iterator ledgerInfoIterator = + ledgers.headMap(slowestReaderLedgerId, false).values().iterator(); + while (ledgerInfoIterator.hasNext()){ + LedgerInfo ls = ledgerInfoIterator.next(); // currentLedger can not be deleted if (ls.getLedgerId() == currentLedger.getId()) { if (log.isDebugEnabled()) { @@ -2560,8 +2567,9 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } totalSizeToDelete += ls.getSize(); - boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(totalSizeToDelete); - boolean expired = hasLedgerRetentionExpired(ls.getTimestamp()); + boolean overRetentionQuota = isLedgerRetentionOverSizeQuota(retentionSizeInMB, totalSizeOfML, + totalSizeToDelete); + boolean expired = hasLedgerRetentionExpired(retentionTimeMs, ls.getTimestamp()); if (log.isDebugEnabled()) { log.debug( "[{}] Checking ledger {} -- time-old: {} sec -- " @@ -2578,14 +2586,19 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) { } ledgersToDelete.add(ls); } else { - if (ls.getLedgerId() < getTheSlowestNonDurationReadPosition().getLedgerId()) { - // once retention constraint has been met, skip check - if (log.isDebugEnabled()) { - log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, - ls.getLedgerId()); - } - invalidateReadHandle(ls.getLedgerId()); + // once retention constraint has been met, skip check + if (log.isDebugEnabled()) { + log.debug("[{}] Ledger {} not deleted. Neither expired nor over-quota", name, ls.getLedgerId()); } + releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue()); + break; + } + } + + while (ledgerInfoIterator.hasNext()) { + LedgerInfo ls = ledgerInfoIterator.next(); + if (!releaseReadHandleIfNoLongerRead(ls.getLedgerId(), slowestNonDurationLedgerId.getValue())) { + break; } } @@ -2686,6 +2699,21 @@ public void operationFailed(MetaStoreException e) { } } + /** + * @param ledgerId the ledger handle which maybe will be released. + * @return if the ledger handle was released. + */ + private boolean releaseReadHandleIfNoLongerRead(long ledgerId, long slowestNonDurationLedgerId) { + if (ledgerId < slowestNonDurationLedgerId) { + if (log.isDebugEnabled()) { + log.debug("[{}] Ledger {} no longer needs to be read, close the cached readHandle", name, ledgerId); + } + invalidateReadHandle(ledgerId); + return true; + } + return false; + } + /** * Non-durable cursors have to be moved forward when data is trimmed since they are not retain that data. * This method also addresses a corner case for durable cursors in which the cursor is caught up, i.e. the mark diff --git a/pom.xml b/pom.xml index 90297c75c2985..9f71cab2f2258 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 pom @@ -28,30 +27,24 @@ apache 23 - - org.apache.pulsar + io.streamnative pulsar - - 2.10.5-SNAPSHOT - + 2.10.5.3 Pulsar Pulsar is a distributed pub-sub messaging platform with a very flexible messaging model and an intuitive client API. https://github.com/apache/pulsar - Apache Software Foundation http://www.apache.org/ 2017 - Apache Pulsar developers http://pulsar.apache.org/ - Apache License, Version 2.0 @@ -59,56 +52,50 @@ flexible messaging model and an intuitive client API. repo - - https://github.com/apache/pulsar - scm:git:https://github.com/apache/pulsar.git - scm:git:ssh://git@github.com:apache/pulsar.git + https://github.com/streamnative/pulsar + scm:git:https://github.com/streamnative/pulsar.git + scm:git:ssh://git@github.com:streamnative/pulsar.git - GitHub Actions https://github.com/apache/pulsar/actions - Github https://github.com/apache/pulsar/issues - 8 8 - **/Test*.java,**/*Test.java,**/*Tests.java,**/*TestCase.java quarantine - UTF-8 UTF-8 true - + true 4 false 1 - + apachepulsar false false package package - 1.21 - 4.14.7 3.6.4 1.5.0 - 1.1.10.1 - 3.2.5 + 1.1.10.1 + + 3.2.5 + 5.1.0 4.1.93.Final 9.4.51.v20230217 @@ -212,13 +199,11 @@ flexible messaging model and an intuitive client API. 1.10.12 5.3.3 2.0.6 - 3.6.0 3.4.0 1.15.3 2.2 - 3.2.8 1.1.1 @@ -234,7 +219,6 @@ flexible messaging model and an intuitive client API. 1.5.4 5.4.0 2.33.2 - 0.6.1 3.0.0 @@ -267,14 +251,11 @@ flexible messaging model and an intuitive client API. 0.4 8.0.1 0.9.44 - rename-netty-native-libs.sh - - org.asynchttpclient async-http-client @@ -290,57 +271,49 @@ flexible messaging model and an intuitive client API. - org.testng testng ${testng.version} - - org.yaml - * - + + org.yaml + * + - org.hamcrest hamcrest ${hamcrest.version} test - junit junit ${junit4.version} - org.awaitility awaitility ${awaitility.version} test - org.mockito mockito-core ${mockito.version} - org.mockito mockito-inline ${mockito.version} - org.powermock powermock-reflect ${powermock.version} - org.apache.zookeeper zookeeper @@ -402,7 +375,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-server @@ -431,13 +403,11 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper cpu-affinity ${bookkeeper.version} - io.vertx vertx-core @@ -448,38 +418,33 @@ flexible messaging model and an intuitive client API. vertx-web ${vertx.version} - - org.apache.curator - curator-recipes - ${curator.version} - - - org.apache.zookeeper - * - - + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + * + + - org.apache.bookkeeper bookkeeper-common-allocator ${bookkeeper.version} - org.apache.bookkeeper bookkeeper-tools-framework ${bookkeeper.version} - org.reflections reflections ${reflections.version} - org.apache.bookkeeper @@ -508,7 +473,6 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper @@ -541,19 +505,16 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper bookkeeper-common ${bookkeeper.version} - org.apache.bookkeeper.stats bookkeeper-stats-api ${bookkeeper.version} - org.apache.bookkeeper.stats datasketches-metrics-provider @@ -565,37 +526,31 @@ flexible messaging model and an intuitive client API. - org.apache.bookkeeper.stats prometheus-metrics-provider ${bookkeeper.version} - org.rocksdb rocksdbjni ${rocksdb.version} - org.eclipse.jetty jetty-server ${jetty.version} - org.eclipse.jetty jetty-alpn-conscrypt-server ${jetty.version} - org.conscrypt conscrypt-openjdk-uber ${conscrypt.version} - org.eclipse.jetty jetty-bom @@ -603,7 +558,6 @@ flexible messaging model and an intuitive client API. pom import - io.netty netty-bom @@ -611,79 +565,66 @@ flexible messaging model and an intuitive client API. pom import - com.beust jcommander ${jcommander.version} - com.google.guava guava ${guava.version} - com.google.inject guice ${guice.version} - com.google.inject.extensions guice-assistedinject ${guice.version} - org.apache.commons commons-lang3 ${commons-lang3.version} - org.apache.commons commons-compress ${commons-compress.version} - commons-configuration commons-configuration ${commons-configuration.version} - commons-io commons-io ${commons-io.version} - org.slf4j slf4j-api ${slf4j.version} - org.slf4j slf4j-simple ${slf4j.version} - org.slf4j jul-to-slf4j ${slf4j.version} - org.slf4j jcl-over-slf4j ${slf4j.version} - org.apache.logging.log4j log4j-bom @@ -691,49 +632,41 @@ flexible messaging model and an intuitive client API. pom import - commons-codec commons-codec ${commons-codec.version} - org.glassfish.jersey.core jersey-server ${jersey.version} - org.glassfish.jersey.core jersey-client ${jersey.version} - org.glassfish.jersey.inject jersey-hk2 ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet-core ${jersey.version} - org.glassfish.jersey.containers jersey-container-servlet ${jersey.version} - javax.ws.rs javax.ws.rs-api ${javax.ws.rs-api.version} - org.glassfish.jersey.media jersey-media-json-jackson @@ -745,23 +678,20 @@ flexible messaging model and an intuitive client API. - org.glassfish.jersey.media jersey-media-multipart ${jersey.version} - net.java.dev.jna jna ${jna.version} - - com.github.docker-java - docker-java-core - ${docker-java.version} + com.github.docker-java + docker-java-core + ${docker-java.version} com.github.docker-java @@ -773,7 +703,6 @@ flexible messaging model and an intuitive client API. docker-java-transport-zerodep ${docker-java.version} - com.fasterxml.jackson jackson-bom @@ -781,122 +710,101 @@ flexible messaging model and an intuitive client API. pom import - org.codehaus.jettison jettison ${jettison.version} - com.fasterxml.woodstox woodstox-core ${woodstox.version} - - org.hdrhistogram HdrHistogram ${hdrHistogram.version} - io.swagger swagger-core ${swagger.version} - io.swagger swagger-annotations ${swagger.version} - javax.servlet javax.servlet-api ${javax.servlet-api} - com.github.ben-manes.caffeine caffeine ${caffeine.version} - com.yahoo.athenz athenz-zts-java-client-core ${athenz.version} - com.yahoo.athenz athenz-zpe-java-client ${athenz.version} - com.github.zafarkhaja java-semver ${java-semver.version} - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_log4j2 ${prometheus.version} - io.prometheus simpleclient_servlet ${prometheus.version} - io.prometheus simpleclient_jetty ${prometheus.version} - io.prometheus simpleclient_caffeine ${prometheus.version} - com.carrotsearch hppc ${hppc.version} - io.etcd jetcd-core ${jetcd.version} - io.etcd jetcd-test ${jetcd.version} - org.apache.spark spark-streaming_2.10 @@ -924,7 +832,6 @@ flexible messaging model and an intuitive client API. - io.jsonwebtoken jjwt-api @@ -940,13 +847,11 @@ flexible messaging model and an intuitive client API. jjwt-jackson ${jsonwebtoken.version} - net.jodah typetools ${typetools.version} - io.grpc grpc-bom @@ -954,7 +859,6 @@ flexible messaging model and an intuitive client API. pom import - io.grpc grpc-all @@ -978,25 +882,21 @@ flexible messaging model and an intuitive client API. - com.google.http-client google-http-client ${google-http-client.version} - com.google.http-client google-http-client-jackson2 ${google-http-client.version} - com.google.http-client google-http-client-gson ${google-http-client.version} - io.perfmark perfmark-api @@ -1009,7 +909,6 @@ flexible messaging model and an intuitive client API. - com.google.protobuf protobuf-bom @@ -1017,19 +916,16 @@ flexible messaging model and an intuitive client API. pom import - com.google.code.gson gson ${gson.version} - com.yahoo.datasketches sketches-core ${sketches.version} - com.amazonaws aws-java-sdk-bom @@ -1037,7 +933,6 @@ flexible messaging model and an intuitive client API. pom import - org.apache.distributedlog distributedlog-core @@ -1050,13 +945,11 @@ flexible messaging model and an intuitive client API. - commons-collections commons-collections ${commons.collections.version} - com.lmax @@ -1080,97 +973,81 @@ flexible messaging model and an intuitive client API. assertj-core ${assertj-core.version} - org.projectlombok lombok ${lombok.version} - javax.annotation javax.annotation-api ${javax.annotation-api.version} - javax.xml.bind jaxb-api ${jaxb-api} - jakarta.xml.bind jakarta.xml.bind-api ${jakarta.xml.bind.version} - com.sun.activation javax.activation ${javax.activation.version} - com.sun.activation jakarta.activation ${jakarta.activation.version} - jakarta.activation jakarta.activation-api ${jakarta.activation.version} - jakarta.validation jakarta.validation-api ${jakarta.validation.version} - io.opencensus opencensus-api ${opencensus.version} - io.opencensus opencensus-contrib-http-util ${opencensus.version} - io.opencensus opencensus-contrib-grpc-metrics ${opencensus.version} - org.opensearch.client opensearch-rest-high-level-client ${opensearch.version} - joda-time joda-time ${joda.version} - org.javassist javassist ${javassist.version} - net.jcip jcip-annotations ${jcip.version} - io.airlift aircompressor @@ -1182,19 +1059,16 @@ flexible messaging model and an intuitive client API. - org.objenesis objenesis ${objenesis.version} - org.apache.httpcomponents httpclient ${apache-http-client.version} - com.github.spotbugs spotbugs-annotations @@ -1202,31 +1076,26 @@ flexible messaging model and an intuitive client API. provided true - com.google.errorprone error_prone_annotations ${errorprone.version} - com.google.j2objc j2objc-annotations ${j2objc-annotations.version} - org.yaml snakeyaml ${snakeyaml.version} - org.apache.ant ant ${ant.version} - com.squareup.okhttp3 okhttp @@ -1247,7 +1116,6 @@ flexible messaging model and an intuitive client API. okio ${okio.version} - org.jetbrains.kotlin kotlin-stdlib @@ -1258,92 +1126,77 @@ flexible messaging model and an intuitive client API. kotlin-stdlib-common ${kotlin-stdlib.version} - org.jetbrains.kotlin kotlin-stdlib-jdk8 ${kotlin-stdlib.version} - - com.typesafe.netty - netty-reactive-streams - ${netty-reactive-streams.version} + com.typesafe.netty + netty-reactive-streams + ${netty-reactive-streams.version} - ch.qos.reload4j reload4j ${reload4j.version} - org.roaringbitmap RoaringBitmap ${roaringbitmap.version} - org.scala-lang scala-library ${scala-library.version} - org.scala-lang scala-reflect ${scala-library.version} - - - org.apache.pulsar + io.streamnative buildtools ${project.version} test - org.testng testng test - org.mockito mockito-core test - org.mockito mockito-inline test - com.github.stefanbirkner system-lambda ${system-lambda.version} test - org.powermock powermock-reflect test - org.assertj assertj-core test - org.projectlombok lombok @@ -1354,7 +1207,6 @@ flexible messaging model and an intuitive client API. javax.annotation-api provided - org.apache.bookkeeper @@ -1364,8 +1216,8 @@ flexible messaging model and an intuitive client API. tests - org.bouncycastle - * + org.bouncycastle + * org.slf4j @@ -1385,8 +1237,12 @@ flexible messaging model and an intuitive client API. + + org.apache.logging.log4j + log4j-layout-template-json + 2.20.0 + - ${project.artifactId} @@ -1395,7 +1251,7 @@ flexible messaging model and an intuitive client API. maven-checkstyle-plugin - org.apache.pulsar + io.streamnative buildtools ${project.version} @@ -1458,7 +1314,6 @@ flexible messaging model and an intuitive client API. - org.commonjava.maven.plugins directory-maven-plugin @@ -1473,14 +1328,13 @@ flexible messaging model and an intuitive client API. pulsar.basedir - org.apache.pulsar + io.streamnative pulsar - com.mycila license-maven-plugin @@ -1591,36 +1445,28 @@ flexible messaging model and an intuitive client API. src/assemble/README.bin.txt src/assemble/LICENSE.bin.txt src/assemble/NOTICE.bin.txt - src/main/java/org/apache/bookkeeper/mledger/proto/MLDataFormats.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java bin/proto/MLDataFormats_pb2.py - **/avro/generated/*.java - **/*.avsc - src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionKey.java src/main/java/org/apache/pulsar/io/kinesis/fbs/KeyValue.java src/main/java/org/apache/pulsar/io/kinesis/fbs/Message.java - src/main/java/org/apache/bookkeeper/mledger/util/AbstractCASReferenceCounted.java - dependency-reduced-pom.xml - pulsar-client-cpp/lib/lz4/lz4.* - pulsar-client-go/go.mod pulsar-client-go/go.sum @@ -1628,20 +1474,16 @@ flexible messaging model and an intuitive client API. pulsar-function-go/go.sum pulsar-function-go/examples/go.mod pulsar-function-go/examples/go.sum - pulsar-client-cpp/lib/checksum/crc32c_sw.cc - pulsar-client-cpp/generated/** **/META-INF/services/com.scurrilous.circe.HashProvider **/META-INF/services/io.prestosql.spi.Plugin - **/django/stats/migrations/*.py **/conf/uwsgi_params - **/*.crt **/*.key @@ -1654,21 +1496,16 @@ flexible messaging model and an intuitive client API. certificate-authority/index.txt certificate-authority/serial certificate-authority/README.md - **/zk-3.5-test-data/* - **/requirements.txt - conf/schema_example.conf **/templates/*.tpl - **/.helmignore **/_helpers.tpl - *.md .github/** @@ -1714,6 +1551,7 @@ flexible messaging model and an intuitive client API. **/*.so.* **/*.dylib src/test/resources/*.txt + **/dependency-reduced-pom.xml @@ -1722,54 +1560,106 @@ flexible messaging model and an intuitive client API. maven-enforcer-plugin ${maven-enforcer-plugin.version} - - enforce-maven - - enforce - - - - - [1.8.0,) - - - [3.3.9,) - - - - + + enforce-maven + + enforce + + + + + [1.8.0,) + + + [3.3.9,) + + + + - org.apache.maven.plugins - maven-assembly-plugin - ${maven-assembly-plugin.version} - false - - - source-release-assembly-tar-gz - generate-sources - - single - - - ${skipSourceReleaseAssembly} - true - - src/assembly-source-package.xml - - apache-pulsar-${project.version}-src - false - - tar.gz - - posix - - - + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + false + + + source-release-assembly-tar-gz + generate-sources + + single + + + ${skipSourceReleaseAssembly} + true + + src/assembly-source-package.xml + + apache-pulsar-${project.version}-src + false + + tar.gz + + posix + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + true + + ossrh + https://s01.oss.sonatype.org/ + 60285aee9d4161 + true + false + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + none + false + + + + + + org.apache.maven.plugins + maven-release-plugin + 3.0.0-M1 + + true + false + release + deploy + - @@ -1787,10 +1677,10 @@ flexible messaging model and an intuitive client API. maven-surefire-plugin - ${include} + ${include} - **/*$*,${exclude} + **/*$*,${exclude} ${groups} ${excludedGroups} @@ -1874,19 +1764,29 @@ flexible messaging model and an intuitive client API. com.github.spotbugs spotbugs-maven-plugin ${spotbugs-maven-plugin.version} - - - com.github.spotbugs - spotbugs - ${spotbugs.version} - - + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + org.codehaus.mojo properties-maven-plugin ${properties-maven-plugin.version} + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + + org.apache.maven.plugins + maven-source-plugin + 3.2.0 + @@ -1902,7 +1802,6 @@ flexible messaging model and an intuitive client API. - jdk11 @@ -2012,7 +1911,6 @@ flexible messaging model and an intuitive client API. tests - contrib-check - - - - org.apache.rat - apache-rat-plugin - - - - check - - verify - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - check-style - verify - - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml - ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml - UTF-8 - **/proto/* - - - check - - - - - - + + + + org.apache.rat + apache-rat-plugin + + + + check + + verify + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + check-style + verify + + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/checkstyle.xml + ${pulsar.basedir}/buildtools/src/main/resources/pulsar/suppressions.xml + UTF-8 + **/proto/* + + + check + + + + + + - windows @@ -2068,7 +1965,6 @@ flexible messaging model and an intuitive client API. rename-netty-native-libs.cmd - @@ -2110,40 +2006,29 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - structured-event-log - pulsar-transaction - pulsar-functions - pulsar-io - kafka-connect-avro-converter-shaded - bouncy-castle - pulsar-client-messagecrypto-bc - pulsar-metadata jclouds-shaded - pulsar-package-management - distribution docker tests - core-modules @@ -2169,31 +2054,23 @@ flexible messaging model and an intuitive client API. pulsar-broker-auth-sasl pulsar-client-auth-sasl pulsar-config-validation - pulsar-transaction - pulsar-functions - pulsar-io - bouncy-castle - pulsar-client-messagecrypto-bc - distribution pulsar-metadata - pulsar-package-management - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-broker-auth-athenz jar Athenz authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - ${project.groupId} testmocks ${project.version} test - com.yahoo.athenz athenz-zpe-java-client - - @@ -74,7 +65,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -92,7 +82,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -106,7 +95,6 @@ - diff --git a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-broker-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-broker-auth-sasl/pom.xml b/pulsar-broker-auth-sasl/pom.xml index 5a4186bd694ad..54833f9bd9ce2 100644 --- a/pulsar-broker-auth-sasl/pom.xml +++ b/pulsar-broker-auth-sasl/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-broker-auth-sasl jar SASL authentication plugin for broker - - ${project.groupId} pulsar-broker ${project.version} - org.apache.kerby kerby-config @@ -53,7 +47,6 @@ - org.apache.kerby kerb-simplekdc @@ -66,30 +59,25 @@ - ${project.groupId} pulsar-proxy ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-client-auth-sasl ${project.version} test - - @@ -111,8 +99,6 @@ - - @@ -132,7 +118,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java index 6ded38132e1ad..9d90cc26ec0a0 100644 --- a/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java +++ b/pulsar-broker-auth-sasl/src/main/java/org/apache/pulsar/broker/authentication/SaslRoleTokenSigner.java @@ -76,7 +76,7 @@ public String verifyAndExtract(String signedStr) throws AuthenticationException String originalSignature = signedStr.substring(index + SIGNATURE.length()); String rawValue = signedStr.substring(0, index); String currentSignature = computeSignature(rawValue); - if (!originalSignature.equals(currentSignature)) { + if (!MessageDigest.isEqual(originalSignature.getBytes(), currentSignature.getBytes())){ throw new AuthenticationException("Invalid signature"); } return rawValue; diff --git a/pulsar-broker-common/pom.xml b/pulsar-broker-common/pom.xml index ca7407c5ec51e..cfef958671cf9 100644 --- a/pulsar-broker-common/pom.xml +++ b/pulsar-broker-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-broker-common Common classes used in multiple broker modules - ${project.groupId} pulsar-metadata ${project.version} - com.google.guava guava - io.prometheus simpleclient_jetty - javax.servlet javax.servlet-api - javax.ws.rs javax.ws.rs-api - io.jsonwebtoken jjwt-impl - io.jsonwebtoken jjwt-jackson - org.bouncycastle @@ -76,14 +65,12 @@ ${bouncycastlefips.version} test - org.awaitility awaitility test - @@ -103,7 +90,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java index bc670c53a8c8f..6f3fb7d714a70 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/BaseResources.java @@ -92,6 +92,13 @@ protected CompletableFuture> getAsync(String path) { return cache.get(path); } + protected CompletableFuture> refreshAndGetAsync(String path) { + return store.sync(path).thenCompose(___ -> { + cache.invalidate(path); + return cache.get(path); + }); + } + protected void set(String path, Function modifyFunction) throws MetadataStoreException { try { setAsync(path, modifyFunction).get(operationTimeoutSec, TimeUnit.SECONDS); diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java index 74d4bcf6c472e..5e59d20bfbfbb 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/resources/NamespaceResources.java @@ -122,6 +122,13 @@ public Optional getPolicies(NamespaceName ns) throws MetadataStoreExce return get(joinPath(BASE_POLICIES_PATH, ns.toString())); } + /** + * Get the namespace policy from the metadata cache. This method will not trigger the load of metadata cache. + * + * @deprecated Since this method may introduce inconsistent namespace policies. we should use + * #{@link NamespaceResources#getPoliciesAsync} + */ + @Deprecated public Optional getPoliciesIfCached(NamespaceName ns) { return getCache().getIfCached(joinPath(BASE_POLICIES_PATH, ns.toString())); } @@ -230,8 +237,18 @@ public CompletableFuture> listPartitionedTopicsAsync(NamespaceName } public CompletableFuture> getPartitionedTopicMetadataAsync(TopicName tn) { - return getAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(), - tn.getEncodedLocalName())); + return getPartitionedTopicMetadataAsync(tn, false); + } + + public CompletableFuture> getPartitionedTopicMetadataAsync(TopicName tn, + boolean refresh) { + if (refresh) { + return refreshAndGetAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(), + tn.getEncodedLocalName())); + } else { + return getAsync(joinPath(PARTITIONED_TOPIC_PATH, tn.getNamespace(), tn.getDomain().value(), + tn.getEncodedLocalName())); + } } public boolean partitionedTopicExists(TopicName tn) throws MetadataStoreException { diff --git a/pulsar-broker-shaded/pom.xml b/pulsar-broker-shaded/pom.xml index 3e5179c8b69c1..91fd0af562af3 100644 --- a/pulsar-broker-shaded/pom.xml +++ b/pulsar-broker-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-broker-shaded jar Pulsar Shaded Broker - ${project.groupId} @@ -41,7 +37,6 @@ ${project.parent.version} - @@ -70,7 +65,6 @@ - maven-antrun-plugin @@ -83,15 +77,12 @@ - + - org.apache.maven.plugins maven-shade-plugin @@ -105,10 +96,9 @@ true true false - - org.apache.pulsar:* + io.streamnative:* org.apache.bookkeeper*:* commons-*:* org.apache.commons:* @@ -119,10 +109,9 @@ com.google.*:* com.fasterxml.jackson.*:* io.netty:* - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:circe-checksum com.yahoo.datasketches:sketches-core - javax.ws.rs:* javax.websocket:* org.glassfish.hk2*:* @@ -138,7 +127,6 @@ org.apache.zookeeper:* jline:* javax.servlet:* - com.beust:* io.swagger:* joda-time:* @@ -154,7 +142,7 @@ - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -205,7 +193,6 @@ com.yahoo.sketches org.apache.pulsar.shade.com.yahoo.sketches - org.apache.zookeeper org.apache.pulsar.shade.org.apache.zookeeper @@ -372,6 +359,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index 4bd6bda8a2af3..d22d56a5e5aed 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-broker jar Pulsar Broker - commons-codec commons-codec - commons-collections commons-collections - org.apache.commons commons-lang3 - org.slf4j slf4j-api - io.netty netty-transport - com.google.protobuf protobuf-java - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - ${project.groupId} managed-ledger ${project.version} - org.apache.curator curator-recipes - org.apache.bookkeeper stream-storage-server @@ -123,242 +109,197 @@ - org.apache.bookkeeper bookkeeper-tools-framework - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-transaction-common ${project.version} - ${project.groupId} pulsar-io-batch-discovery-triggerers ${project.version} test - ${project.groupId} testmocks ${project.version} test - com.github.tomakehurst wiremock-jre8 ${wiremock.version} test - - io.dropwizard.metrics - metrics-core + io.dropwizard.metrics + metrics-core - - org.xerial.snappy - snappy-java + org.xerial.snappy + snappy-java - - ${project.groupId} pulsar-functions-worker ${project.version} - ${project.groupId} pulsar-functions-local-runner-original ${project.version} test - ${project.groupId} pulsar-client-messagecrypto-bc ${project.version} test - org.awaitility awaitility test - - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.media jersey-media-json-jackson - org.glassfish.jersey.test-framework jersey-test-framework-core test ${jersey.version} - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 test ${jersey.version} - jakarta.activation jakarta.activation-api - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - com.fasterxml.jackson.module jackson-module-jsonSchema - org.slf4j jul-to-slf4j - org.slf4j jcl-over-slf4j - com.google.guava guava - com.beust jcommander - io.swagger swagger-annotations - io.prometheus simpleclient - io.prometheus simpleclient_jetty - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_caffeine - io.swagger swagger-core - org.hdrhistogram HdrHistogram - com.google.code.gson gson - com.github.zafarkhaja java-semver - org.apache.avro avro ${avro.version} - com.carrotsearch hppc - org.roaringbitmap RoaringBitmap - ${project.groupId} pulsar-functions-api-examples @@ -366,7 +307,6 @@ pom test - ${project.groupId} pulsar-io-batch-data-generator @@ -374,7 +314,6 @@ pom test - ${project.groupId} pulsar-io-data-generator @@ -382,7 +321,6 @@ pom test - ${project.groupId} pulsar-metadata @@ -390,7 +328,6 @@ test-jar test - javax.xml.bind jaxb-api @@ -401,35 +338,28 @@ - com.sun.activation javax.activation - - ${project.groupId} pulsar-transaction-coordinator ${project.version} - - ${project.groupId} pulsar-package-core ${project.version} - io.etcd jetcd-test test - @@ -462,7 +392,6 @@ - maven-dependency-plugin @@ -506,7 +435,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -521,7 +449,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin @@ -618,7 +545,6 @@ - @@ -637,7 +563,6 @@ test-jar test - ${project.groupId} pulsar-package-core @@ -645,7 +570,6 @@ test-jar test - ${project.groupId} pulsar-metadata @@ -793,7 +717,7 @@ - + diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index da37d1b7e545c..f4e5caacded63 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -2208,9 +2208,10 @@ protected void internalCreateSubscription(AsyncResponse asyncResponse, String su internalCreateSubscriptionForNonPartitionedTopic(asyncResponse, subscriptionName, targetMessageId, authoritative, replicated, properties); } else { - boolean allowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName); - getPartitionedTopicMetadataAsync(topicName, - authoritative, allowAutoTopicCreation).thenAccept(partitionMetadata -> { + pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName) + .thenCompose(allowAutoTopicCreation -> + getPartitionedTopicMetadataAsync(topicName, authoritative, allowAutoTopicCreation)) + .thenAccept(partitionMetadata -> { final int numPartitions = partitionMetadata.partitions; if (numPartitions > 0) { final CompletableFuture future = new CompletableFuture<>(); @@ -2306,13 +2307,13 @@ private void internalCreateSubscriptionForNonPartitionedTopic( MessageIdImpl targetMessageId, boolean authoritative, boolean replicated, Map properties) { - boolean isAllowAutoTopicCreation = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName); - - validateTopicOwnershipAsync(topicName, authoritative) - .thenCompose(__ -> { - validateTopicOperation(topicName, TopicOperation.SUBSCRIBE, subscriptionName); - return pulsar().getBrokerService().getTopic(topicName.toString(), isAllowAutoTopicCreation); - }).thenApply(optTopic -> { + pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName) + .thenCompose(isAllowAutoTopicCreation -> validateTopicOwnershipAsync(topicName, authoritative) + .thenCompose(__ -> { + validateTopicOperation(topicName, TopicOperation.SUBSCRIBE, subscriptionName); + return pulsar().getBrokerService().getTopic(topicName.toString(), isAllowAutoTopicCreation); + })) + .thenApply(optTopic -> { if (optTopic.isPresent()) { return optTopic.get(); } else { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java index e0127bd38646f..26c31a6e8a955 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/LoadManagerShared.java @@ -554,4 +554,10 @@ public static void filterBrokersWithLargeTopicCount(Set brokerCandidateC brokerCandidateCache.addAll(filteredBrokerCandidates); } } + + public static NamespaceBundle getNamespaceBundle(PulsarService pulsar, String bundle) { + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + return pulsar.getNamespaceService().getNamespaceBundleFactory().getBundle(namespaceName, bundleRange); + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java index 074083052895e..301f19213c352 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImpl.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance.impl; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; @@ -642,9 +643,21 @@ public synchronized void doLoadShedding() { if (!shouldAntiAffinityNamespaceUnload(namespaceName, bundleRange, broker)) { return; } + NamespaceBundle bundleToUnload = LoadManagerShared.getNamespaceBundle(pulsar, bundle); + Optional destBroker = this.selectBroker(bundleToUnload); + if (!destBroker.isPresent()) { + log.info("[{}] No broker available to unload bundle {} from broker {}", + strategy.getClass().getSimpleName(), bundle, broker); + return; + } + if (destBroker.get().equals(broker)) { + log.warn("[{}] The destination broker {} is the same as the current owner broker for Bundle {}", + strategy.getClass().getSimpleName(), destBroker.get(), bundle); + return; + } - log.info("[{}] Unloading bundle: {} from broker {}", - strategy.getClass().getSimpleName(), bundle, broker); + log.info("[{}] Unloading bundle: {} from broker {} to dest broker {}", + strategy.getClass().getSimpleName(), bundle, broker, destBroker.get()); try { pulsar.getAdminClient().namespaces().unloadNamespaceBundle(namespaceName, bundleRange); loadData.getRecentlyUnloadedBundles().put(bundle, System.currentTimeMillis()); @@ -799,16 +812,56 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni // If the given bundle is already in preallocated, return the selected broker. return Optional.of(preallocatedBundleToBroker.get(bundle)); } - final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, - key -> getBundleDataOrDefault(bundle)); - brokerCandidateCache.clear(); - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - // filter brokers which owns topic higher than threshold - LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, - conf.getLoadBalancerBrokerMaxTopics()); + Optional broker = selectBroker(serviceUnit); + if (!broker.isPresent()) { + // If no broker is selected, return empty. + return broker; + } + // Add new bundle to preallocated. + preallocateBundle(bundle, broker.get()); + return broker; + } + } finally { + selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + } + } + + private void preallocateBundle(String bundle, String broker) { + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + loadData.getBrokerData().get(broker).getPreallocatedBundleData().put(bundle, data); + preallocatedBundleToBroker.put(bundle, broker); + + final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); + final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); + final ConcurrentOpenHashMap> namespaceToBundleRange = + brokerToNamespaceToBundleRange + .computeIfAbsent(broker, + k -> ConcurrentOpenHashMap.>newBuilder() + .build()); + synchronized (namespaceToBundleRange) { + namespaceToBundleRange.computeIfAbsent(namespaceName, + k -> ConcurrentOpenHashSet.newBuilder().build()) + .add(bundleRange); + } + } + + @VisibleForTesting + Optional selectBroker(final ServiceUnitId serviceUnit) { + synchronized (brokerCandidateCache) { + final String bundle = serviceUnit.toString(); + final BundleData data = loadData.getBundleData().computeIfAbsent(bundle, + key -> getBundleDataOrDefault(bundle)); + brokerCandidateCache.clear(); + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + + // filter brokers which owns topic higher than threshold + LoadManagerShared.filterBrokersWithLargeTopicCount(brokerCandidateCache, loadData, + conf.getLoadBalancerBrokerMaxTopics()); // distribute namespaces to domain and brokers according to anti-affinity-group LoadManagerShared.filterAntiAffinityGroupOwnedBrokers(pulsar, serviceUnit.toString(), @@ -820,71 +873,50 @@ public Optional selectBrokerForAssignment(final ServiceUnitId serviceUni brokerToNamespaceToBundleRange); log.info("{} brokers being considered for assignment of {}", brokerCandidateCache.size(), bundle); - // Use the filter pipeline to finalize broker candidates. - try { - for (BrokerFilter filter : filterPipeline) { - filter.filter(brokerCandidateCache, data, loadData, conf); - } - } catch (BrokerFilterException x) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - } - - if (brokerCandidateCache.isEmpty()) { - // restore the list of brokers to the full set - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); + // Use the filter pipeline to finalize broker candidates. + try { + for (BrokerFilter filter : filterPipeline) { + filter.filter(brokerCandidateCache, data, loadData, conf); } + } catch (BrokerFilterException x) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } - // Choose a broker among the potentially smaller filtered list, when possible - Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); - if (log.isDebugEnabled()) { - log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); - } + if (brokerCandidateCache.isEmpty()) { + // restore the list of brokers to the full set + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + } - if (!broker.isPresent()) { - // No brokers available - return broker; - } + // Choose a broker among the potentially smaller filtered list, when possible + Optional broker = placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (log.isDebugEnabled()) { + log.debug("Selected broker {} from candidate brokers {}", broker, brokerCandidateCache); + } - final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; - final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage(); - if (maxUsage > overloadThreshold) { - // All brokers that were in the filtered list were overloaded, so check if there is a better broker - LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, - getAvailableBrokers(), - brokerTopicLoadingPredicate); - Optional brokerTmp = - placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); - if (brokerTmp.isPresent()) { - broker = brokerTmp; - } - } + if (!broker.isPresent()) { + // No brokers available + return broker; + } - // Add new bundle to preallocated. - loadData.getBrokerData().get(broker.get()).getPreallocatedBundleData().put(bundle, data); - preallocatedBundleToBroker.put(bundle, broker.get()); - - final String namespaceName = LoadManagerShared.getNamespaceNameFromBundleName(bundle); - final String bundleRange = LoadManagerShared.getBundleRangeFromBundleName(bundle); - final ConcurrentOpenHashMap> namespaceToBundleRange = - brokerToNamespaceToBundleRange - .computeIfAbsent(broker.get(), - k -> ConcurrentOpenHashMap.>newBuilder() - .build()); - synchronized (namespaceToBundleRange) { - namespaceToBundleRange.computeIfAbsent(namespaceName, - k -> ConcurrentOpenHashSet.newBuilder().build()) - .add(bundleRange); + final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; + final double maxUsage = loadData.getBrokerData().get(broker.get()).getLocalData().getMaxResourceUsage(); + if (maxUsage > overloadThreshold) { + // All brokers that were in the filtered list were overloaded, so check if there is a better broker + LoadManagerShared.applyNamespacePolicies(serviceUnit, policies, brokerCandidateCache, + getAvailableBrokers(), + brokerTopicLoadingPredicate); + Optional brokerTmp = + placementStrategy.selectBroker(brokerCandidateCache, data, loadData, conf); + if (brokerTmp.isPresent()) { + broker = brokerTmp; } - return broker; } - } finally { - selectBrokerForAssignment.observe(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + return broker; } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java index ae1d2a5bab02b..7baebc7d1b6d4 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/lookup/TopicLookupBase.java @@ -83,9 +83,11 @@ protected void internalLookupTopicAsync(TopicName topicName, boolean authoritati // Currently, it's hard to check the non-persistent-non-partitioned topic, because it only exists in the broker, // it doesn't have metadata. If the topic is non-persistent and non-partitioned, we'll return the true flag. - CompletableFuture existFuture = pulsar().getBrokerService().isAllowAutoTopicCreation(topicName) - || (!topicName.isPersistent() && !topicName.isPartitioned()) - ? CompletableFuture.completedFuture(true) : pulsar().getNamespaceService().checkTopicExists(topicName); + CompletableFuture existFuture = pulsar().getBrokerService().isAllowAutoTopicCreationAsync(topicName) + .thenCompose(isAllowAutoTopicCreation -> + isAllowAutoTopicCreation || (!topicName.isPersistent() && !topicName.isPartitioned()) + ? CompletableFuture.completedFuture(true) : + pulsar().getNamespaceService().checkTopicExists(topicName)); existFuture.thenAccept(exist -> { if (!exist) { completeLookupResponseExceptionally(asyncResponse, new RestException(Response.Status.NOT_FOUND, diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java index 736fcf1f5e1e5..76e1df0d93d7f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/AbstractTopic.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; import static org.apache.bookkeeper.mledger.impl.ManagedLedgerMBeanImpl.ENTRY_LATENCY_BUCKETS_USEC; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; @@ -40,6 +41,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.ToLongFunction; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.bookkeeper.mledger.util.StatsBuckets; import org.apache.commons.collections4.CollectionUtils; @@ -57,6 +59,7 @@ import org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException; import org.apache.pulsar.broker.stats.prometheus.metrics.Summary; import org.apache.pulsar.common.api.proto.CommandSubscribe.SubType; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.BacklogQuota; import org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies; @@ -131,7 +134,7 @@ public abstract class AbstractTopic implements Topic, TopicPolicyListener USER_CREATED_PRODUCER_COUNTER_UPDATER = AtomicIntegerFieldUpdater.newUpdater(AbstractTopic.class, "userCreatedProducerCount"); - private volatile int userCreatedProducerCount = 0; + protected volatile int userCreatedProducerCount = 0; protected volatile Optional topicEpoch = Optional.empty(); private volatile boolean hasExclusiveProducer; @@ -1065,6 +1068,12 @@ public PublishRateLimiter getBrokerPublishRateLimiter() { return brokerService.getBrokerPublishRateLimiter(); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and we can use + * #{@link AbstractTopic#updateResourceGroupLimiter(Policies)} to instead of it. + */ + @Deprecated public void updateResourceGroupLimiter(Optional optPolicies) { Policies policies; try { @@ -1078,17 +1087,20 @@ public void updateResourceGroupLimiter(Optional optPolicies) { log.warn("[{}] Error getting policies {} and publish throttling will be disabled", topic, e.getMessage()); policies = new Policies(); } + updateResourceGroupLimiter(policies); + } + public void updateResourceGroupLimiter(@Nonnull Policies namespacePolicies) { + requireNonNull(namespacePolicies); // attach the resource-group level rate limiters, if set - String rgName = policies.resource_group_name; + String rgName = namespacePolicies.resource_group_name; if (rgName != null) { final ResourceGroup resourceGroup = - brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); + brokerService.getPulsar().getResourceGroupServiceManager().resourceGroupGet(rgName); if (resourceGroup != null) { this.resourceGroupRateLimitingEnabled = true; this.resourceGroupPublishLimiter = resourceGroup.getResourceGroupPublishLimiter(); - this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), - () -> this.enableCnxAutoRead()); + this.resourceGroupPublishLimiter.registerRateLimitFunction(this.getName(), this::enableCnxAutoRead); log.info("Using resource group {} rate limiter for topic {}", rgName, topic); return; } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index cac70f6896e5f..08550886ecb4a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun; import static org.apache.commons.collections.CollectionUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -56,6 +57,7 @@ import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -72,6 +74,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.function.Predicate; +import javax.annotation.Nonnull; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -931,7 +934,15 @@ public CompletableFuture> getTopicIfExists(final String topic) { } public CompletableFuture getOrCreateTopic(final String topic) { - return getTopic(topic, isAllowAutoTopicCreation(topic)).thenApply(Optional::get); + final TopicName topicName; + try { + topicName = TopicName.get(topic); + } catch (Throwable ex) { + return FutureUtil.failedFuture(ex); + } + return isAllowAutoTopicCreationAsync(topicName) + .thenCompose(isAllowAutoTopicCreation -> getTopic(topic, isAllowAutoTopicCreation) + .thenApply(Optional::get)); } public CompletableFuture> getTopic(final String topic, boolean createIfMissing) { @@ -2704,12 +2715,9 @@ public CompletableFuture fetchPartitionedTopicMetadata if (pulsar.getNamespaceService() == null) { return FutureUtil.failedFuture(new NamingException("namespace service is not ready")); } - Optional policies = - pulsar.getPulsarResources().getNamespaceResources() - .getPoliciesIfCached(topicName.getNamespaceObject()); - return pulsar.getNamespaceService().checkTopicExists(topicName) - .thenCompose(topicExists -> { - return fetchPartitionedTopicMetadataAsync(topicName) + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenCompose(policies -> pulsar.getNamespaceService().checkTopicExists(topicName) + .thenCompose(topicExists -> fetchPartitionedTopicMetadataAsync(topicName) .thenCompose(metadata -> { CompletableFuture future = new CompletableFuture<>(); @@ -2721,7 +2729,8 @@ public CompletableFuture fetchPartitionedTopicMetadata if (metadata.partitions == 0 && !topicExists && !topicName.isPartitioned() - && pulsar.getBrokerService().isAllowAutoTopicCreation(topicName, policies) + && pulsar.getBrokerService() + .isAllowAutoTopicCreation(topicName, policies) && pulsar.getBrokerService() .isDefaultTopicTypePartitioned(topicName, policies)) { @@ -2729,10 +2738,13 @@ public CompletableFuture fetchPartitionedTopicMetadata .createDefaultPartitionedTopicAsync(topicName, policies) .thenAccept(md -> future.complete(md)) .exceptionally(ex -> { + log.info("[{}] The partitioned topic is already" + + " created, try to refresh the cache and read" + + " again.", topicName); if (ex.getCause() instanceof MetadataStoreException.AlreadyExistsException) { // The partitioned topic might be created concurrently - fetchPartitionedTopicMetadataAsync(topicName) + fetchPartitionedTopicMetadataAsync(topicName, true) .whenComplete((metadata2, ex2) -> { if (ex2 == null) { future.complete(metadata2); @@ -2741,6 +2753,9 @@ public CompletableFuture fetchPartitionedTopicMetadata } }); } else { + log.error("[{}] operation of creating partitioned" + + " topic metadata failed", + topicName, ex); future.completeExceptionally(ex); } return null; @@ -2751,8 +2766,8 @@ public CompletableFuture fetchPartitionedTopicMetadata }); return future; - }); - }); + })) + ); } @SuppressWarnings("deprecation") @@ -2780,9 +2795,14 @@ private CompletableFuture createDefaultPartitionedTopi } public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName) { + return fetchPartitionedTopicMetadataAsync(topicName, false); + } + + public CompletableFuture fetchPartitionedTopicMetadataAsync(TopicName topicName, + boolean refreshCacheAndGet) { // gets the number of partitions from the configuration cache return pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources() - .getPartitionedTopicMetadataAsync(topicName).thenApply(metadata -> { + .getPartitionedTopicMetadataAsync(topicName, refreshCacheAndGet).thenApply(metadata -> { // if the partitioned topic is not found in metadata, then the topic is not partitioned return metadata.orElseGet(() -> new PartitionedTopicMetadata()); }); @@ -2947,11 +2967,23 @@ private void foreachCnx(Consumer consumer) { cnxSet.forEach(consumer); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} + * You can use #{@link BrokerService#isAllowAutoTopicCreationAsync(TopicName)} + */ + @Deprecated public boolean isAllowAutoTopicCreation(final String topic) { TopicName topicName = TopicName.get(topic); return isAllowAutoTopicCreation(topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} + * You can use #{@link BrokerService#isAllowAutoTopicCreationAsync(TopicName)} + */ + @Deprecated public boolean isAllowAutoTopicCreation(final TopicName topicName) { Optional policies = pulsar.getPulsarResources().getNamespaceResources() @@ -2959,6 +2991,11 @@ public boolean isAllowAutoTopicCreation(final TopicName topicName) { return isAllowAutoTopicCreation(topicName, policies); } + public CompletableFuture isAllowAutoTopicCreationAsync(final TopicName topicName) { + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(topicName.getNamespaceObject()) + .thenApply(policies -> isAllowAutoTopicCreation(topicName, policies)); + } + public boolean isAllowAutoTopicCreation(final TopicName topicName, final Optional policies) { if (policies.isPresent() && policies.get().deleted) { log.info("Preventing AutoTopicCreation on a namespace that is being deleted {}", @@ -3005,11 +3042,23 @@ private AutoTopicCreationOverride getAutoTopicCreationOverride(final TopicName t return null; } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated public boolean isAllowAutoSubscriptionCreation(final String topic) { TopicName topicName = TopicName.get(topic); return isAllowAutoSubscriptionCreation(topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) { AutoSubscriptionCreationOverride autoSubscriptionCreationOverride = getAutoSubscriptionCreationOverride(topicName); @@ -3020,6 +3069,12 @@ public boolean isAllowAutoSubscriptionCreation(final TopicName topicName) { } } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link BrokerService#isAllowAutoSubscriptionCreationAsync(TopicName)} to instead of it. + */ + @Deprecated private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(final TopicName topicName) { Optional policies = pulsar.getPulsarResources().getNamespaceResources().getPoliciesIfCached(topicName.getNamespaceObject()); @@ -3031,6 +3086,19 @@ private AutoSubscriptionCreationOverride getAutoSubscriptionCreationOverride(fin return null; } + public @Nonnull CompletionStage isAllowAutoSubscriptionCreationAsync(@Nonnull TopicName tpName) { + requireNonNull(tpName); + // namespace level policies + return pulsar.getPulsarResources().getNamespaceResources().getPoliciesAsync(tpName.getNamespaceObject()) + .thenApply(policies -> { + if (policies.isPresent() && policies.get().autoSubscriptionCreationOverride != null) { + return policies.get().autoSubscriptionCreationOverride.isAllowAutoSubscriptionCreation(); + } + // broker level policies + return pulsar.getConfiguration().isAllowAutoSubscriptionCreation(); + }); + } + public boolean isSystemTopic(String topic) { return isSystemTopic(TopicName.get(topic)); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index f2cd77e485e23..8924b750eb624 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -318,6 +318,12 @@ public Future sendMessages(final List entries, EntryBatchSizes batc msgOutCounter.add(totalMessages); bytesOutCounter.add(totalBytes); chunkedMessageRate.recordMultipleEvents(totalChunkedMessages, 0); + } else { + if (log.isDebugEnabled()) { + log.debug("[{}-{}] Sent messages to client fail by IO exception[{}], close the connection" + + " immediately. Consumer: {}", topicName, subscription, + status.cause() == null ? "" : status.cause().getMessage(), this.toString()); + } } }); return writeAndFlushPromise; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java index c1045c3d14b73..ef7cfdf14d87d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/ServerCnx.java @@ -53,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -994,7 +995,8 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { remoteAddress, getPrincipal()); } - log.info("[{}] Subscribing on topic {} / {}", remoteAddress, topicName, subscriptionName); + log.info("[{}] Subscribing on topic {} / {}. consumerId: {}", this.ctx().channel().toString(), + topicName, subscriptionName, consumerId); try { Metadata.validateMetadata(metadata, service.getPulsar().getConfiguration().getMaxConsumerMetadataSize()); @@ -1033,48 +1035,53 @@ protected void handleSubscribe(final CommandSubscribe subscribe) { return null; } - boolean createTopicIfDoesNotExist = forceTopicCreation - && service.isAllowAutoTopicCreation(topicName.toString()); - - service.getTopic(topicName.toString(), createTopicIfDoesNotExist) + service.isAllowAutoTopicCreationAsync(topicName) + .thenCompose(isAllowAutoTopicCreation -> { + final boolean createTopicIfDoesNotExist = forceTopicCreation && isAllowAutoTopicCreation; + return service.getTopic(topicName.toString(), createTopicIfDoesNotExist); + }) .thenCompose(optTopic -> { if (!optTopic.isPresent()) { return FutureUtil .failedFuture(new TopicNotFoundException( "Topic " + topicName + " does not exist")); } - - Topic topic = optTopic.get(); - - boolean rejectSubscriptionIfDoesNotExist = isDurable - && !service.isAllowAutoSubscriptionCreation(topicName.toString()) - && !topic.getSubscriptions().containsKey(subscriptionName) - && topic.isPersistent(); - - if (rejectSubscriptionIfDoesNotExist) { - return FutureUtil - .failedFuture( - new SubscriptionNotFoundException( - "Subscription does not exist")); - } - - SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) - .subscriptionName(subscriptionName) - .consumerId(consumerId).subType(subType).priorityLevel(priorityLevel) - .consumerName(consumerName).isDurable(isDurable) - .startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted) - .initialPosition(initialPosition) - .startMessageRollbackDurationSec(startMessageRollbackDurationSec) - .replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta) - .subscriptionProperties(subscriptionProperties) - .consumerEpoch(consumerEpoch) - .build(); - if (schema != null) { - return topic.addSchemaIfIdleOrCheckCompatible(schema) - .thenCompose(v -> topic.subscribe(option)); - } else { - return topic.subscribe(option); - } + final Topic topic = optTopic.get(); + return service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowAutoSubscriptionCreation -> { + boolean rejectSubscriptionIfDoesNotExist = isDurable + && !isAllowAutoSubscriptionCreation + && !topic.getSubscriptions().containsKey(subscriptionName) + && topic.isPersistent(); + + if (rejectSubscriptionIfDoesNotExist) { + return FutureUtil + .failedFuture( + new SubscriptionNotFoundException( + "Subscription does not exist")); + } + + SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this) + .subscriptionName(subscriptionName) + .consumerId(consumerId).subType(subType) + .priorityLevel(priorityLevel) + .consumerName(consumerName).isDurable(isDurable) + .startMessageId(startMessageId).metadata(metadata) + .readCompacted(readCompacted) + .initialPosition(initialPosition) + .startMessageRollbackDurationSec(startMessageRollbackDurationSec) + .replicatedSubscriptionStateArg(isReplicated) + .keySharedMeta(keySharedMeta) + .subscriptionProperties(subscriptionProperties) + .consumerEpoch(consumerEpoch) + .build(); + if (schema != null) { + return topic.addSchemaIfIdleOrCheckCompatible(schema) + .thenCompose(v -> topic.subscribe(option)); + } else { + return topic.subscribe(option); + } + }); }) .thenAccept(consumer -> { if (consumerFuture.complete(consumer)) { @@ -1285,33 +1292,39 @@ protected void handleProducer(final CommandProducer cmdProducer) { schemaVersionFuture.thenAccept(schemaVersion -> { topic.checkIfTransactionBufferRecoverCompletely(isTxnEnabled).thenAccept(future -> { - CompletableFuture createInitSubFuture; + CompletionStage createInitSubFuture; if (!Strings.isNullOrEmpty(initialSubscriptionName) && topic.isPersistent() && !topic.getSubscriptions().containsKey(initialSubscriptionName)) { - if (!this.getBrokerService().isAllowAutoSubscriptionCreation(topicName)) { - String msg = - "Could not create the initial subscription due to the auto subscription " - + "creation is not allowed."; - if (producerFuture.completeExceptionally( - new BrokerServiceException.NotAllowedException(msg))) { - log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", - remoteAddress, msg, initialSubscriptionName, topicName); - commandSender.sendErrorResponse(requestId, - ServerError.NotAllowedError, msg); - } - producers.remove(producerId, producerFuture); - return; - } - createInitSubFuture = - topic.createSubscription(initialSubscriptionName, InitialPosition.Earliest, - false, null); + createInitSubFuture = service.isAllowAutoSubscriptionCreationAsync(topicName) + .thenCompose(isAllowAutoSubscriptionCreation -> { + if (!isAllowAutoSubscriptionCreation) { + return FutureUtil.failedFuture( + new BrokerServiceException.NotAllowedException( + "Could not create the initial subscription due to" + + " the auto subscription creation is not allowed.")); + } + return topic.createSubscription(initialSubscriptionName, + InitialPosition.Earliest, false, null); + }); } else { createInitSubFuture = CompletableFuture.completedFuture(null); } createInitSubFuture.whenComplete((sub, ex) -> { if (ex != null) { + final Throwable rc = FutureUtil.unwrapCompletionException(ex); + if (rc instanceof BrokerServiceException.NotAllowedException) { + log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", + remoteAddress, rc.getMessage(), initialSubscriptionName, topicName); + if (producerFuture.completeExceptionally(rc)) { + commandSender.sendErrorResponse(requestId, + ServerError.NotAllowedError, rc.getMessage()); + } + producers.remove(producerId, producerFuture); + return; + } + String msg = "Failed to create the initial subscription: " + ex.getCause().getMessage(); log.warn("[{}] {} initialSubscriptionName: {}, topic: {}", @@ -1537,11 +1550,12 @@ private void printSendCommandDebug(CommandSend send, ByteBuf headersAndPayload) headersAndPayload.resetReaderIndex(); if (log.isDebugEnabled()) { log.debug("[{}] Received send message request. producer: {}:{} {}:{} size: {}," - + " partition key is: {}, ordering key is {}", + + " partition key is: {}, ordering key is {}, uncompressedSize is {}", remoteAddress, send.getProducerId(), send.getSequenceId(), msgMetadata.getProducerName(), msgMetadata.getSequenceId(), headersAndPayload.readableBytes(), msgMetadata.hasPartitionKey() ? msgMetadata.getPartitionKey() : null, - msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null); + msgMetadata.hasOrderingKey() ? msgMetadata.getOrderingKey() : null, + msgMetadata.getUncompressedSize()); } } @@ -1573,6 +1587,12 @@ protected void handleAck(CommandAck ack) { } return null; }); + } else { + if (log.isDebugEnabled()) { + log.debug("Consumer future is not complete(not complete or error), but received command ack. so discard" + + " this command. consumerId: {}, cnx: {}, messageIdCount: {}", ack.getConsumerId(), + this.ctx().channel().toString(), ack.getMessageIdsCount()); + } } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java index c171fea0a6f97..419c6d88b22a1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesService.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nonnull; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.namespace.NamespaceBundleOwnershipListener; @@ -220,12 +221,25 @@ public TopicPolicies getTopicPolicies(TopicName topicName, NamespaceName namespace = topicName.getNamespaceObject(); prepareInitPoliciesCache(namespace, new CompletableFuture<>()); } - if (policyCacheInitMap.containsKey(topicName.getNamespaceObject()) - && !policyCacheInitMap.get(topicName.getNamespaceObject())) { - throw new TopicPoliciesCacheNotInitException(); + + MutablePair result = new MutablePair<>(); + policyCacheInitMap.compute(topicName.getNamespaceObject(), (k, initialized) -> { + if (initialized == null || !initialized) { + result.setLeft(new TopicPoliciesCacheNotInitException()); + } else { + TopicPolicies topicPolicies = + isGlobal ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) + : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); + result.setRight(topicPolicies); + } + return initialized; + }); + + if (result.getLeft() != null) { + throw result.getLeft(); + } else { + return result.getRight(); } - return isGlobal ? globalPoliciesCache.get(TopicName.get(topicName.getPartitionedTopicName())) - : policiesCache.get(TopicName.get(topicName.getPartitionedTopicName())); } @Override @@ -389,7 +403,7 @@ private void initPolicesCache(SystemTopicClient.Reader reader, Comp private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean cleanOwnedBundlesCount) { CompletableFuture> readerFuture = readerCaches.remove(namespace); - policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + if (cleanOwnedBundlesCount) { ownedBundlesCountPerNamespace.remove(namespace); } @@ -400,7 +414,11 @@ private void cleanCacheAndCloseReader(@Nonnull NamespaceName namespace, boolean return null; }); } - policyCacheInitMap.remove(namespace); + + policyCacheInitMap.compute(namespace, (k, v) -> { + policiesCache.entrySet().removeIf(entry -> Objects.equals(entry.getKey().getNamespaceObject(), namespace)); + return null; + }); } private void readMorePolicies(SystemTopicClient.Reader reader) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java index 54f738101be27..a021e7ce94def 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/nonpersistent/NonPersistentTopic.java @@ -157,18 +157,20 @@ public CompletableFuture initialize() { return brokerService.pulsar().getPulsarResources().getNamespaceResources() .getPoliciesAsync(TopicName.get(topic).getNamespaceObject()) .thenAccept(optPolicies -> { + final Policies policies; if (!optPolicies.isPresent()) { log.warn("[{}] Policies not present and isEncryptionRequired will be set to false", topic); isEncryptionRequired = false; + policies = new Policies(); } else { - Policies policies = optPolicies.get(); + policies = optPolicies.get(); updateTopicPolicyByNamespacePolicy(policies); isEncryptionRequired = policies.encryption_required; isAllowAutoUpdateSchema = policies.is_allow_auto_update_schema; schemaValidationEnforced = policies.schema_validation_enforced; } updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(policies); }); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java index cd7f5e9ea64c4..568ae0915cb3d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/DispatchRateLimiter.java @@ -364,6 +364,12 @@ public static CompletableFuture> getPoliciesAsync(BrokerServi return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesAsync(namespace); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. we can use #{@link DispatchRateLimiter#getPoliciesAsync(BrokerService, String)} to instead of it. + */ + @Deprecated public static Optional getPolicies(BrokerService brokerService, String topicName) { final NamespaceName namespace = TopicName.get(topicName).getNamespaceObject(); return brokerService.pulsar().getPulsarResources().getNamespaceResources().getPoliciesIfCached(namespace); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java index e40e7585f1637..4c3d8e1a467c5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/MessageDeduplication.java @@ -55,6 +55,8 @@ public class MessageDeduplication { private final ManagedLedger managedLedger; private ManagedCursor managedCursor; + private static final String IS_LAST_CHUNK = "isLastChunk"; + enum Status { // Deduplication is initialized @@ -324,11 +326,12 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade String producerName = publishContext.getProducerName(); long sequenceId = publishContext.getSequenceId(); long highestSequenceId = Math.max(publishContext.getHighestSequenceId(), sequenceId); + MessageMetadata md = null; if (producerName.startsWith(replicatorPrefix)) { // Message is coming from replication, we need to use the original producer name and sequence id // for the purpose of deduplication and not rely on the "replicator" name. int readerIndex = headersAndPayload.readerIndex(); - MessageMetadata md = Commands.parseMessageMetadata(headersAndPayload); + md = Commands.parseMessageMetadata(headersAndPayload); producerName = md.getProducerName(); sequenceId = md.getSequenceId(); highestSequenceId = Math.max(md.getHighestSequenceId(), sequenceId); @@ -337,7 +340,23 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade publishContext.setOriginalHighestSequenceId(highestSequenceId); headersAndPayload.readerIndex(readerIndex); } - + long chunkID = -1; + long totalChunk = -1; + if (publishContext.isChunked()) { + if (md == null) { + int readerIndex = headersAndPayload.readerIndex(); + md = Commands.parseMessageMetadata(headersAndPayload); + headersAndPayload.readerIndex(readerIndex); + } + chunkID = md.getChunkId(); + totalChunk = md.getNumChunksFromMsg(); + } + // All chunks of a message use the same message metadata and sequence ID, + // so we only need to check the sequence ID for the last chunk in a chunk message. + if (chunkID != -1 && chunkID != totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.FALSE); + return MessageDupStatus.NotDup; + } // Synchronize the get() and subsequent put() on the map. This would only be relevant if the producer // disconnects and re-connects very quickly. At that point the call can be coming from a different thread synchronized (highestSequencedPushed) { @@ -363,6 +382,11 @@ public MessageDupStatus isDuplicate(PublishContext publishContext, ByteBuf heade } highestSequencedPushed.put(producerName, highestSequenceId); } + // Only put sequence ID into highestSequencedPushed and + // highestSequencedPersisted until receive and persistent the last chunk. + if (chunkID != -1 && chunkID == totalChunk - 1) { + publishContext.setProperty(IS_LAST_CHUNK, Boolean.TRUE); + } return MessageDupStatus.NotDup; } @@ -383,8 +407,10 @@ public void recordMessagePersisted(PublishContext publishContext, PositionImpl p sequenceId = publishContext.getOriginalSequenceId(); highestSequenceId = publishContext.getOriginalHighestSequenceId(); } - - highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + Boolean isLastChunk = (Boolean) publishContext.getProperty(IS_LAST_CHUNK); + if (isLastChunk == null || isLastChunk) { + highestSequencedPersisted.put(producerName, Math.max(highestSequenceId, sequenceId)); + } if (++snapshotCounter >= snapshotInterval) { snapshotCounter = 0; takeSnapshot(position); @@ -482,6 +508,10 @@ public long getLastPublishedSequenceId(String producerName) { } public void takeSnapshot() { + if (!isEnabled()) { + return; + } + Integer interval = topic.getHierarchyTopicPolicies().getDeduplicationSnapshotIntervalSeconds().get(); long currentTimeStamp = System.currentTimeMillis(); if (interval == null || interval <= 0 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index 5054b039af79b..14074fc280bd6 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service.persistent; import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled; import static org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames; @@ -49,6 +50,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import lombok.Getter; import org.apache.bookkeeper.client.api.LedgerMetadata; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -316,7 +318,7 @@ public CompletableFuture initialize() { if (!optPolicies.isPresent()) { isEncryptionRequired = false; updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(new Policies()); initializeDispatchRateLimiterIfNeeded(); updateSubscribeRateLimiter(); return; @@ -332,7 +334,7 @@ public CompletableFuture initialize() { updatePublishDispatcher(); - updateResourceGroupLimiter(optPolicies); + updateResourceGroupLimiter(policies); this.isEncryptionRequired = policies.encryption_required; @@ -2419,7 +2421,8 @@ public void updateDispatchRateLimiter() { } @Override - public CompletableFuture onPoliciesUpdate(Policies data) { + public CompletableFuture onPoliciesUpdate(@Nonnull Policies data) { + requireNonNull(data); if (log.isDebugEnabled()) { log.debug("[{}] isEncryptionRequired changes: {} -> {}", topic, isEncryptionRequired, data.encryption_required); @@ -2443,7 +2446,7 @@ public CompletableFuture onPoliciesUpdate(Policies data) { updatePublishDispatcher(); - this.updateResourceGroupLimiter(Optional.of(data)); + updateResourceGroupLimiter(data); List> producerCheckFutures = new ArrayList<>(producers.size()); producers.values().forEach(producer -> producerCheckFutures.add( @@ -2783,6 +2786,11 @@ public synchronized void triggerCompaction() throws PulsarServerException, AlreadyRunningException { if (currentCompaction.isDone()) { currentCompaction = brokerService.pulsar().getCompactor().compact(topic); + currentCompaction.whenComplete((ignore, ex) -> { + if (ex != null){ + log.warn("[{}] Compaction failure.", topic, ex); + } + }); } else { throw new AlreadyRunningException("Compaction already in progress"); } @@ -2860,7 +2868,7 @@ public CompletableFuture addSchemaIfIdleOrCheckCompatible(SchemaData schem .mapToInt(subscription -> subscription.getConsumers().size()) .sum(); if (hasSchema - || (!producers.isEmpty()) + || (userCreatedProducerCount > 0) || (numActiveConsumers != 0) || (ledger.getTotalSize() != 0)) { return checkSchemaCompatibleForConsumer(schema); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java index 1e1245ed36b6f..cf1603788f1fd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/ReplicatedSubscriptionsController.java @@ -192,10 +192,14 @@ private void receiveSubscriptionUpdated(ReplicatedSubscriptionsUpdate update) { sub.acknowledgeMessage(Collections.singletonList(pos), AckType.Cumulative, Collections.emptyMap()); } else { // Subscription doesn't exist. We need to force the creation of the subscription in this cluster, because - log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subcription", + log.info("[{}][{}] Creating subscription at {}:{} after receiving update from replicated subscription", topic, update.getSubscriptionName(), updatedMessageId.getLedgerId(), pos); - topic.createSubscription(update.getSubscriptionName(), - InitialPosition.Latest, true /* replicateSubscriptionState */, null); + topic.createSubscription(update.getSubscriptionName(), InitialPosition.Earliest, + true /* replicateSubscriptionState */, Collections.emptyMap()) + .thenAccept(subscriptionCreated -> { + subscriptionCreated.acknowledgeMessage(Collections.singletonList(pos), + AckType.Cumulative, Collections.emptyMap()); + }); } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java index 89af6f6be882f..f602659400f00 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/SubscribeRateLimiter.java @@ -28,6 +28,7 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.policies.data.Policies; import org.apache.pulsar.common.policies.data.SubscribeRate; import org.apache.pulsar.common.util.RateLimiter; @@ -159,14 +160,21 @@ public void onSubscribeRateUpdate(SubscribeRate subscribeRate) { } /** - * Gets configured subscribe-rate from namespace policies. Returns null if subscribe-rate is not configured - * - * @return + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. */ + @Deprecated public SubscribeRate getPoliciesSubscribeRate() { return getPoliciesSubscribeRate(brokerService, topicName); } + /** + * @deprecated Avoid using the deprecated method + * #{@link org.apache.pulsar.broker.resources.NamespaceResources#getPoliciesIfCached(NamespaceName)} and blocking + * call. + */ + @Deprecated public static SubscribeRate getPoliciesSubscribeRate(BrokerService brokerService, final String topicName) { final String cluster = brokerService.pulsar().getConfiguration().getClusterName(); final Optional policies = DispatchRateLimiter.getPolicies(brokerService, topicName); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java index 2521bc9e754ac..c096d3610667e 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawBatchConverter.java @@ -42,10 +42,20 @@ public class RawBatchConverter { public static boolean isReadableBatch(RawMessage msg) { ByteBuf payload = msg.getHeadersAndPayload(); MessageMetadata metadata = Commands.parseMessageMetadata(payload); + return isReadableBatch(metadata); + } + + public static boolean isReadableBatch(MessageMetadata metadata) { return metadata.hasNumMessagesInBatch() && metadata.getEncryptionKeysCount() == 0; } public static List> extractIdsAndKeysAndSize(RawMessage msg) + throws IOException { + return extractIdsAndKeysAndSize(msg, true); + } + + public static List> extractIdsAndKeysAndSize(RawMessage msg, + boolean extractNullKey) throws IOException { checkArgument(msg.getMessageIdData().getBatchIndex() == -1); @@ -69,7 +79,7 @@ public static List> extractIdsAndKey msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i); - if (!smm.isCompactedOut()) { + if (!smm.isCompactedOut() && (extractNullKey || smm.hasPartitionKey())) { idsAndKeysAndSize.add(ImmutableTriple.of(id, smm.hasPartitionKey() ? smm.getPartitionKey() : null, smm.hasPayloadSize() ? smm.getPayloadSize() : 0)); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java index 2431d14209816..7188ee6aa88bf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactedTopicImpl.java @@ -119,7 +119,7 @@ public void asyncReadEntriesOrWait(ManagedCursor cursor, return CompletableFuture.completedFuture(null); } else { long endPoint = Math.min(context.ledger.getLastAddConfirmed(), - startPoint + numberOfEntriesToRead); + startPoint + (numberOfEntriesToRead - 1)); if (startPoint == NEWER_THAN_COMPACTED) { cursor.seek(compactionHorizon.getNext()); callback.readEntriesComplete(Collections.emptyList(), readEntriesCtx); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index ded798c49d5dd..5d9ae3cf9a163 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -127,22 +127,28 @@ private void phaseOneLoop(RawReader reader, boolean deletedMessage = false; boolean replaceMessage = false; mxBean.addCompactionReadOp(reader.getTopic(), m.getHeadersAndPayload().readableBytes()); - if (RawBatchConverter.isReadableBatch(m)) { + MessageMetadata metadata = Commands.parseMessageMetadata(m.getHeadersAndPayload()); + if (RawBatchConverter.isReadableBatch(metadata)) { try { + int numMessagesInBatch = metadata.getNumMessagesInBatch(); + int deleteCnt = 0; for (ImmutableTriple e : RawBatchConverter - .extractIdsAndKeysAndSize(m)) { + .extractIdsAndKeysAndSize(m, false)) { if (e != null) { if (e.getRight() > 0) { MessageId old = latestForKey.put(e.getMiddle(), e.getLeft()); - replaceMessage = old != null; + if (old != null) { + mxBean.addCompactionRemovedEvent(reader.getTopic()); + } } else { - deletedMessage = true; latestForKey.remove(e.getMiddle()); + deleteCnt++; + mxBean.addCompactionRemovedEvent(reader.getTopic()); } } - if (replaceMessage || deletedMessage) { - mxBean.addCompactionRemovedEvent(reader.getTopic()); - } + } + if (deleteCnt == numMessagesInBatch) { + deletedMessage = true; } } catch (IOException ioe) { log.info("Error decoding batch for message {}. Whole batch will be included in output", @@ -273,11 +279,14 @@ private void phaseTwoLoop(RawReader reader, MessageId to, Map } }); if (to.equals(id)) { + // make sure all inflight writes have finished + outstanding.acquire(MAX_OUTSTANDING); addFuture.whenComplete((res, exception2) -> { if (exception2 == null) { promise.complete(null); } }); + return; } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java index 36f27877360d8..f3e3188de4bac 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicPoliciesTest.java @@ -2953,6 +2953,7 @@ public void testLoopCreateAndDeleteTopicPolicies() throws Exception { }); } } + @Test public void testGlobalTopicPolicies() throws Exception { final String topic = testTopic + UUID.randomUUID(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java similarity index 91% rename from pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java rename to pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 6b5898f4174e4..dae160b1421f0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.broker.loadbalance; +package org.apache.pulsar.broker.loadbalance.impl; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -51,12 +53,12 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.broker.loadbalance.LoadBalancerTestingUtils; +import org.apache.pulsar.broker.loadbalance.LoadData; +import org.apache.pulsar.broker.loadbalance.LoadManager; import org.apache.pulsar.broker.TimeAverageMessageData; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared; import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared.BrokerTopicLoadingPredicate; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; -import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper; -import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceAllocationPolicies; import org.apache.pulsar.client.admin.Namespaces; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.common.naming.NamespaceBundle; @@ -329,38 +331,70 @@ public void testLoadShedding() throws Exception { bundleReference.set(invocation.getArguments()[0].toString() + '/' + invocation.getArguments()[1]); return null; }).when(namespacesSpy1).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + + AtomicReference> selectedBrokerRef = new AtomicReference<>(); + ModularLoadManagerImpl primaryLoadManagerSpy = spy(primaryLoadManager); + doAnswer(invocation -> { + ServiceUnitId serviceUnitId = (ServiceUnitId) invocation.getArguments()[0]; + Optional broker = primaryLoadManager.selectBroker(serviceUnitId); + selectedBrokerRef.set(broker); + return broker; + }).when(primaryLoadManagerSpy).selectBroker(any()); + setField(pulsar1.getAdminClient(), "namespaces", namespacesSpy1); pulsar1.getConfiguration().setLoadBalancerEnabled(true); - final LoadData loadData = (LoadData) getField(primaryLoadManager, "loadData"); + final LoadData loadData = (LoadData) getField(primaryLoadManagerSpy, "loadData"); final Map brokerDataMap = loadData.getBrokerData(); final BrokerData brokerDataSpy1 = spy(brokerDataMap.get(primaryHost)); when(brokerDataSpy1.getLocalData()).thenReturn(localBrokerData); brokerDataMap.put(primaryHost, brokerDataSpy1); // Need to update all the bundle data for the shredder to see the spy. - primaryLoadManager.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); + primaryLoadManagerSpy.handleDataNotification(new Notification(NotificationType.Created, LoadManager.LOADBALANCE_BROKERS_ROOT + "/broker:8080")); Thread.sleep(100); localBrokerData.setCpu(new ResourceUsage(80, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // 80% is below overload threshold: verify nothing is unloaded. - verify(namespacesSpy1, Mockito.times(0)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(0)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); localBrokerData.setCpu(new ResourceUsage(90, 100)); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Most expensive bundle will be unloaded. - verify(namespacesSpy1, Mockito.times(1)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(1)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(2)); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now less expensive bundle will be unloaded (normally other bundle would move off and nothing would be // unloaded, but this is not the case due to the spy's behavior). - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); assertEquals(bundleReference.get(), mockBundleName(1)); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); - primaryLoadManager.doLoadShedding(); + primaryLoadManagerSpy.doLoadShedding(); // Now both are in grace period: neither should be unloaded. - verify(namespacesSpy1, Mockito.times(2)).unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + verify(namespacesSpy1, Mockito.times(2)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + assertEquals(selectedBrokerRef.get().get(), secondaryHost); + + // Test bundle transfer to same broker + + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + verify(namespacesSpy1, Mockito.times(3)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + + doReturn(Optional.of(primaryHost)).when(primaryLoadManagerSpy).selectBroker(any()); + loadData.getRecentlyUnloadedBundles().clear(); + primaryLoadManagerSpy.doLoadShedding(); + // The bundle shouldn't be unloaded because the broker is the same. + verify(namespacesSpy1, Mockito.times(3)) + .unloadNamespaceBundle(Mockito.anyString(), Mockito.anyString()); + } // Test that ModularLoadManagerImpl will determine that writing local data to ZooKeeper is necessary if certain diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java index 5ae6076274826..5906c6f4fe216 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/lookup/http/HttpTopicLookupv2Test.java @@ -142,6 +142,9 @@ public void testLookupTopicNotExist() throws Exception { doReturn(uri).when(uriInfo).getRequestUri(); doReturn(true).when(config).isAuthorizationEnabled(); + BrokerService brokerService = pulsar.getBrokerService(); + doReturn(CompletableFuture.completedFuture(false)) + .when(brokerService).isAllowAutoTopicCreationAsync(any()); NamespaceService namespaceService = pulsar.getNamespaceService(); CompletableFuture future = new CompletableFuture<>(); future.complete(false); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java new file mode 100644 index 0000000000000..3110120c6946e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BkEnsemblesChaosTest.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.client.api.Producer; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test(groups = "broker") +public class BkEnsemblesChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testBookieInfoIsCorrectEvenIfLostNotificationDueToZKClientReconnect() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + final byte[] msgValue = "test".getBytes(); + admin.topics().createNonPartitionedTopic(topicName); + // Ensure broker works. + Producer producer1 = client.newProducer().topic(topicName).create(); + producer1.send(msgValue); + producer1.close(); + admin.topics().unload(topicName); + + // Restart some bookies, which triggers the ZK node of Bookie deleted and created. + // And make the local metadata store reconnect to lose some notification of the ZK node change. + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.stopBK(i); + } + makeLocalMetadataStoreKeepReconnect(); + for (int i = 0; i < numberOfBookies - 1; i++){ + bkEnsemble.startBK(i); + } + // Sleep 100ms to lose the notifications of ZK node create. + Thread.sleep(100); + stopLocalMetadataStoreAlwaysReconnect(); + + // Ensure broker still works. + admin.topics().unload(topicName); + Producer producer2 = client.newProducer().topic(topicName).create(); + producer2.send(msgValue); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java new file mode 100644 index 0000000000000..614b4f0837022 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceChaosTest.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import static org.testng.Assert.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.partition.PartitionedTopicMetadata; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.common.policies.data.impl.AutoTopicCreationOverrideImpl; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker") +public class BrokerServiceChaosTest extends CanReconnectZKClientPulsarServiceBaseTest { + + @Override + @BeforeClass(alwaysRun = true, timeOut = 300000) + public void setup() throws Exception { + super.setup(); + } + + @Override + @AfterClass(alwaysRun = true, timeOut = 300000) + public void cleanup() throws Exception { + super.cleanup(); + } + + @Test + public void testFetchPartitionedTopicMetadataWithCacheRefresh() throws Exception { + final String configMetadataStoreConnectString = + WhiteboxImpl.getInternalState(pulsar.getConfigurationMetadataStore(), "zkConnectString"); + final ZooKeeper anotherZKCli = new ZooKeeper(configMetadataStoreConnectString, 5000, null); + // Set policy of auto create topic to PARTITIONED. + final String ns = defaultTenant + "/ns_" + UUID.randomUUID().toString().replaceAll("-", ""); + final TopicName topicName1 = TopicName.get("persistent://" + ns + "/tp1"); + final TopicName topicName2 = TopicName.get("persistent://" + ns + "/tp2"); + admin.namespaces().createNamespace(ns); + AutoTopicCreationOverride autoTopicCreationOverride = + new AutoTopicCreationOverrideImpl.AutoTopicCreationOverrideImplBuilder().allowAutoTopicCreation(true) + .topicType(TopicType.PARTITIONED.toString()) + .defaultNumPartitions(3).build(); + admin.namespaces().setAutoTopicCreationAsync(ns, autoTopicCreationOverride); + // Make the cache of namespace policy is valid. + admin.namespaces().getAutoSubscriptionCreation(ns); + // Trigger the zk node "/admin/partitioned-topics/{namespace}/persistent" created. + admin.topics().createPartitionedTopic(topicName1.toString(), 2); + admin.topics().deletePartitionedTopic(topicName1.toString()); + + // Since there is no partitioned metadata created, the partitions count of metadata will be 0. + PartitionedTopicMetadata partitionedTopicMetadata1 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + assertEquals(partitionedTopicMetadata1.partitions, 0); + + // Create the partitioned metadata by another zk client. + // Make a error to make the cache could not update. + makeLocalMetadataStoreKeepReconnect(); + anotherZKCli.create("/admin/partitioned-topics/" + ns + "/persistent/" + topicName2.getLocalName(), + "{\"partitions\":3}".getBytes(StandardCharsets.UTF_8), + ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + stopLocalMetadataStoreAlwaysReconnect(); + + // Get the partitioned metadata from cache, there is 90% chance that partitions count of metadata is 0. + PartitionedTopicMetadata partitionedTopicMetadata2 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2).get(); + // Note: If you want to reproduce the issue, you can perform validation on the next line. + // assertEquals(partitionedTopicMetadata2.partitions, 0); + + // Verify the new method will return a correct result. + PartitionedTopicMetadata partitionedTopicMetadata3 = + pulsar.getBrokerService().fetchPartitionedTopicMetadataAsync(topicName2, true).get(); + assertEquals(partitionedTopicMetadata3.partitions, 3); + + // cleanup. + admin.topics().deletePartitionedTopic(topicName2.toString()); + anotherZKCli.close(); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index e6c12bbc5785f..579db85dc249b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -1157,7 +1157,7 @@ public void testConcurrentLoadTopicExceedLimitShouldNotBeAutoCreated() throws Ex } else { // check topic should not be created if disable autoCreateTopic. Assert.assertTrue(load.isDone()); - Assert.assertTrue(load.get().isPresent()); + Assert.assertFalse(load.get().isPresent()); } } } finally { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java new file mode 100644 index 0000000000000..49fd1eb051009 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java @@ -0,0 +1,215 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import com.google.common.collect.Sets; +import io.netty.channel.Channel; +import java.net.URL; +import java.nio.channels.SelectionKey; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.client.admin.PulsarAdmin; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.common.policies.data.ClusterData; +import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.policies.data.TopicType; +import org.apache.pulsar.metadata.impl.ZKMetadataStore; +import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.pulsar.zookeeper.ZookeeperServerTest; +import org.apache.zookeeper.ClientCnxn; +import org.apache.zookeeper.ZooKeeper; +import org.awaitility.reflect.WhiteboxImpl; + +@Slf4j +public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetrySupport { + + protected final String defaultTenant = "public"; + protected final String defaultNamespace = defaultTenant + "/default"; + protected int numberOfBookies = 3; + protected final String clusterName = "r1"; + protected URL url; + protected URL urlTls; + protected ServiceConfiguration config = new ServiceConfiguration(); + protected ZookeeperServerTest brokerConfigZk; + protected LocalBookkeeperEnsemble bkEnsemble; + protected PulsarService pulsar; + protected BrokerService broker; + protected PulsarAdmin admin; + protected PulsarClient client; + protected ZooKeeper localZkOfBroker; + protected Object localMetaDataStoreClientCnx; + protected final AtomicBoolean LocalMetadataStoreInReconnectFinishSignal = new AtomicBoolean(); + protected void startZKAndBK() throws Exception { + // Start ZK. + brokerConfigZk = new ZookeeperServerTest(0); + brokerConfigZk.start(); + + // Start BK. + bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0); + bkEnsemble.start(); + } + + protected void startBrokers() throws Exception { + // Start brokers. + setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk); + pulsar = new PulsarService(config); + pulsar.start(); + broker = pulsar.getBrokerService(); + ZKMetadataStore zkMetadataStore = (ZKMetadataStore) pulsar.getLocalMetadataStore(); + localZkOfBroker = zkMetadataStore.getZkClient(); + ClientCnxn cnxn = WhiteboxImpl.getInternalState(localZkOfBroker, "cnxn"); + Object sendThread = WhiteboxImpl.getInternalState(cnxn, "sendThread"); + localMetaDataStoreClientCnx = WhiteboxImpl.getInternalState(sendThread, "clientCnxnSocket"); + + url = new URL(pulsar.getWebServiceAddress()); + urlTls = new URL(pulsar.getWebServiceAddressTls()); + admin = PulsarAdmin.builder().serviceHttpUrl(url.toString()).build(); + client = PulsarClient.builder().serviceUrl(url.toString()).build(); + } + + protected void makeLocalMetadataStoreKeepReconnect() throws Exception { + if (!LocalMetadataStoreInReconnectFinishSignal.compareAndSet(false, true)) { + throw new RuntimeException("Local metadata store is already keeping reconnect"); + } + if (localMetaDataStoreClientCnx.getClass().getSimpleName().equals("ClientCnxnSocketNIO")) { + makeLocalMetadataStoreKeepReconnectNIO(); + } else { + // ClientCnxnSocketNetty. + makeLocalMetadataStoreKeepReconnectNetty(); + } + } + + protected void makeLocalMetadataStoreKeepReconnectNIO() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + SelectionKey sockKey = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "sockKey"); + if (sockKey != null) { + sockKey.channel().close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void makeLocalMetadataStoreKeepReconnectNetty() { + new Thread(() -> { + while (LocalMetadataStoreInReconnectFinishSignal.get()) { + try { + Channel channel = WhiteboxImpl.getInternalState(localMetaDataStoreClientCnx, "channel"); + if (channel != null) { + channel.close(); + } + // Prevents high cpu usage. + Thread.sleep(5); + } catch (Exception e) { + log.error("Try close the ZK connection of local metadata store failed: {}", e.toString()); + } + } + }).start(); + } + + protected void stopLocalMetadataStoreAlwaysReconnect() { + LocalMetadataStoreInReconnectFinishSignal.set(false); + } + + protected void createDefaultTenantsAndClustersAndNamespace() throws Exception { + admin.clusters().createCluster(clusterName, ClusterData.builder() + .serviceUrl(url.toString()) + .serviceUrlTls(urlTls.toString()) + .brokerServiceUrl(pulsar.getBrokerServiceUrl()) + .brokerServiceUrlTls(pulsar.getBrokerServiceUrlTls()) + .brokerClientTlsEnabled(false) + .build()); + + admin.tenants().createTenant(defaultTenant, new TenantInfoImpl(Collections.emptySet(), + Sets.newHashSet(clusterName))); + + admin.namespaces().createNamespace(defaultNamespace, Sets.newHashSet(clusterName)); + } + + @Override + protected void setup() throws Exception { + incrementSetupNumber(); + + log.info("--- Starting OneWayReplicatorTestBase::setup ---"); + + startZKAndBK(); + + startBrokers(); + + createDefaultTenantsAndClustersAndNamespace(); + + Thread.sleep(100); + log.info("--- OneWayReplicatorTestBase::setup completed ---"); + } + + private void setConfigDefaults(ServiceConfiguration config, String clusterName, + LocalBookkeeperEnsemble bookkeeperEnsemble, ZookeeperServerTest brokerConfigZk) { + config.setClusterName(clusterName); + config.setAdvertisedAddress("localhost"); + config.setWebServicePort(Optional.of(0)); + config.setWebServicePortTls(Optional.of(0)); + config.setMetadataStoreUrl("zk:127.0.0.1:" + bookkeeperEnsemble.getZookeeperPort()); + config.setConfigurationMetadataStoreUrl("zk:127.0.0.1:" + brokerConfigZk.getZookeeperPort() + "/foo"); + config.setBrokerDeleteInactiveTopicsEnabled(false); + config.setBrokerDeleteInactiveTopicsFrequencySeconds(60); + config.setBrokerShutdownTimeoutMs(0L); + config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); + config.setBrokerServicePort(Optional.of(0)); + config.setBrokerServicePortTls(Optional.of(0)); + config.setBacklogQuotaCheckIntervalInSeconds(5); + config.setDefaultNumberOfNamespaceBundles(1); + config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED.toString()); + config.setEnableReplicatedSubscriptions(true); + config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + } + + @Override + protected void cleanup() throws Exception { + markCurrentSetupNumberCleaned(); + log.info("--- Shutting down ---"); + + stopLocalMetadataStoreAlwaysReconnect(); + + // Stop brokers. + client.close(); + admin.close(); + if (pulsar != null) { + pulsar.close(); + } + + // Stop ZK and BK. + bkEnsemble.stop(); + brokerConfigZk.stop(); + + // Reset configs. + config = new ServiceConfiguration(); + setConfigDefaults(config, clusterName, bkEnsemble, brokerConfigZk); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java index 83f4040aa40e7..73a8aca13a9a0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/OneWayReplicatorTest.java @@ -19,13 +19,19 @@ package org.apache.pulsar.broker.service; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import java.util.Optional; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.policies.data.TopicStats; import org.junit.Assert; +import org.awaitility.Awaitility; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -75,4 +81,29 @@ public void testReplicatorProducerStatInTopic() throws Exception { admin2.topics().delete(topicName); }); } + + @Test + public void testCreateRemoteConsumerFirst() throws Exception { + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + defaultNamespace + "/tp_"); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).create(); + // Wait for replicator started. + Awaitility.await().untilAsserted(() -> { + Optional topicOptional2 = pulsar2.getBrokerService().getTopic(topicName, false).get(); + assertTrue(topicOptional2.isPresent()); + PersistentTopic persistentTopic2 = (PersistentTopic) topicOptional2.get(); + assertFalse(persistentTopic2.getProducers().isEmpty()); + }); + // The topic in cluster2 has a replicator created producer(schema Auto_Produce), but does not have any schema。 + // Verify: the consumer of this cluster2 can create successfully. + Consumer consumer2 = client2.newConsumer(Schema.STRING).topic(topicName).subscriptionName("s1") + .subscribe();; + + // cleanup. + producer1.close(); + consumer2.close(); + cleanupTopics(() -> { + admin1.topics().delete(topicName); + admin2.topics().delete(topicName); + }); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java index 046adaa5ec257..250d971b9fb95 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ReplicatorSubscriptionTest.java @@ -24,10 +24,12 @@ import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; @@ -41,6 +43,7 @@ import org.apache.pulsar.broker.service.persistent.ReplicatedSubscriptionsController; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; @@ -48,6 +51,7 @@ import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.common.policies.data.PartitionedTopicStats; import org.apache.pulsar.common.policies.data.TopicStats; +import org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap; import org.awaitility.Awaitility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -154,6 +158,93 @@ public void testReplicatedSubscriptionAcrossTwoRegions() throws Exception { "messages don't match."); } + @Test + public void testReplicatedSubscribeAndSwitchToStandbyCluster() throws Exception { + final String namespace = BrokerTestUtil.newUniqueName("pulsar/ns_"); + final String topicName = BrokerTestUtil.newUniqueName("persistent://" + namespace + "/tp_"); + final String subscriptionName = "s1"; + final boolean isReplicatedSubscription = true; + final int messagesCount = 20; + final LinkedHashSet sentMessages = new LinkedHashSet<>(); + final Set receivedMessages = Collections.synchronizedSet(new LinkedHashSet<>()); + admin1.namespaces().createNamespace(namespace); + admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2")); + admin1.topics().createNonPartitionedTopic(topicName); + admin1.topics().createSubscription(topicName, subscriptionName, MessageId.earliest, isReplicatedSubscription); + final PersistentTopic topic1 = + (PersistentTopic) pulsar1.getBrokerService().getTopic(topicName, false).join().get(); + + // Send messages + // Wait for the topic created on the cluster2. + // Wait for the snapshot created. + final PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).build(); + Producer producer1 = client1.newProducer(Schema.STRING).topic(topicName).enableBatching(false).create(); + Consumer consumer1 = client1.newConsumer(Schema.STRING).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + for (int i = 0; i < messagesCount / 2; i++) { + String msg = i + ""; + producer1.send(msg); + sentMessages.add(msg); + } + Awaitility.await().untilAsserted(() -> { + ConcurrentOpenHashMap replicators = topic1.getReplicators(); + assertTrue(replicators != null && replicators.size() == 1, "Replicator should started"); + assertTrue(replicators.values().iterator().next().isConnected(), "Replicator should be connected"); + assertTrue(topic1.getReplicatedSubscriptionController().get().getLastCompletedSnapshotId().isPresent(), + "One snapshot should be finished"); + }); + final PersistentTopic topic2 = + (PersistentTopic) pulsar2.getBrokerService().getTopic(topicName, false).join().get(); + Awaitility.await().untilAsserted(() -> { + assertTrue(topic2.getReplicatedSubscriptionController().isPresent(), + "Replicated subscription controller should created"); + }); + for (int i = messagesCount / 2; i < messagesCount; i++) { + String msg = i + ""; + producer1.send(msg); + sentMessages.add(msg); + } + + // Consume half messages and wait the subscription created on the cluster2. + for (int i = 0; i < messagesCount / 2; i++){ + Message message = consumer1.receive(2, TimeUnit.SECONDS); + if (message == null) { + fail("Should not receive null."); + } + receivedMessages.add(message.getValue()); + consumer1.acknowledge(message); + } + Awaitility.await().untilAsserted(() -> { + assertNotNull(topic2.getSubscriptions().get(subscriptionName), "Subscription should created"); + }); + + // Switch client to cluster2. + // Since the cluster1 was not crash, all messages will be replicated to the cluster2. + consumer1.close(); + final PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).build(); + final Consumer consumer2 = client2.newConsumer(Schema.AUTO_CONSUME()).topic(topicName) + .subscriptionName(subscriptionName).replicateSubscriptionState(isReplicatedSubscription).subscribe(); + + // Verify all messages will be consumed. + Awaitility.await().untilAsserted(() -> { + while (true) { + Message message = consumer2.receive(2, TimeUnit.SECONDS); + if (message != null) { + receivedMessages.add(message.getValue().toString()); + consumer2.acknowledge(message); + } else { + break; + } + } + assertEquals(receivedMessages.size(), sentMessages.size()); + }); + + consumer2.close(); + producer1.close(); + client1.close(); + client2.close(); + } + /** * If there's no traffic, the snapshot creation should stop and then resume when traffic comes back */ diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index e06c256ccb558..c90c4a3a5bc31 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -33,10 +33,13 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerServiceException.TopicPoliciesCacheNotInitException; import org.apache.pulsar.broker.systopic.NamespaceEventsSystemTopicFactory; @@ -53,13 +56,16 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicPolicies; import org.apache.pulsar.common.util.FutureUtil; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; +import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @Test(groups = "broker") +@Slf4j public class SystemTopicBasedTopicPoliciesServiceTest extends MockedPulsarServiceBaseTest { private static final String NAMESPACE1 = "system-topic/namespace-1"; @@ -385,4 +391,71 @@ public void testHandleNamespaceBeingDeleted() throws Exception { }); service.deleteTopicPoliciesAsync(TOPIC1).get(); } + + @Test + public void testGetTopicPoliciesWithCleanCache() throws Exception { + final String topic = "persistent://" + NAMESPACE1 + "/test" + UUID.randomUUID(); + pulsarClient.newProducer().topic(topic).create().close(); + + SystemTopicBasedTopicPoliciesService topicPoliciesService = + (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); + + ConcurrentHashMap spyPoliciesCache = spy(new ConcurrentHashMap()); + FieldUtils.writeDeclaredField(topicPoliciesService, "policiesCache", spyPoliciesCache, true); + + Awaitility.await().untilAsserted(() -> { + Assertions.assertThat(topicPoliciesService.getTopicPolicies(TopicName.get(topic))).isNull(); + }); + + admin.topicPolicies().setMaxConsumersPerSubscription(topic, 1); + Awaitility.await().untilAsserted(() -> { + Assertions.assertThat(pulsar.getTopicPoliciesService().getTopicPolicies(TopicName.get(topic))).isNotNull(); + }); + + Map>> readers = + (Map>>) + FieldUtils.readDeclaredField(topicPoliciesService, "readerCaches", true); + + Mockito.doAnswer(invocation -> { + Thread.sleep(1000); + return invocation.callRealMethod(); + }).when(spyPoliciesCache).get(Mockito.any()); + + CompletableFuture result = new CompletableFuture<>(); + Thread thread = new Thread(() -> { + TopicPolicies topicPolicies; + for (int i = 0; i < 10; i++) { + try { + topicPolicies = topicPoliciesService.getTopicPolicies(TopicName.get(topic)); + Assert.assertNotNull(topicPolicies); + Thread.sleep(500); + } catch (BrokerServiceException.TopicPoliciesCacheNotInitException e) { + log.warn("topic policies cache not init, retry..."); + } catch (Throwable e) { + log.error("ops: ", e); + result.completeExceptionally(e); + return; + } + } + result.complete(null); + }); + + Thread thread2 = new Thread(() -> { + for (int i = 0; i < 10; i++) { + CompletableFuture> readerCompletableFuture = + readers.get(TopicName.get(topic).getNamespaceObject()); + if (readerCompletableFuture != null) { + readerCompletableFuture.join().closeAsync().join(); + } + } + }); + + thread.start(); + thread2.start(); + + thread.join(); + thread2.join(); + + result.join(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java index c7cf44cb07acc..93590bb4ff583 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/persistent/TopicDuplicationTest.java @@ -505,6 +505,43 @@ public void testDisableNamespacePolicyTakeSnapshot() throws Exception { } + @Test(timeOut = 30000) + public void testDisableNamespacePolicyTakeSnapshotShouldNotThrowException() throws Exception { + cleanup(); + conf.setBrokerDeduplicationEnabled(true); + conf.setBrokerDeduplicationSnapshotFrequencyInSeconds(1); + conf.setBrokerDeduplicationSnapshotIntervalSeconds(1); + conf.setBrokerDeduplicationEntriesInterval(20000); + setup(); + + final String topicName = testTopic + UUID.randomUUID().toString(); + final String producerName = "my-producer"; + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING).topic(topicName).enableBatching(false).producerName(producerName).create(); + + // disable deduplication + admin.namespaces().setDeduplicationStatus(myNamespace, false); + + int msgNum = 50; + CountDownLatch countDownLatch = new CountDownLatch(msgNum); + for (int i = 0; i < msgNum; i++) { + producer.newMessage().value("msg" + i).sendAsync().whenComplete((res, e) -> countDownLatch.countDown()); + } + countDownLatch.await(); + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().get(); + ManagedCursor managedCursor = persistentTopic.getMessageDeduplication().getManagedCursor(); + + // when disable topic deduplication the cursor should be deleted. + assertNull(managedCursor); + + // this method will be called at brokerService forEachTopic. + // if topic level disable deduplication. + // this method should be skipped without throw exception. + persistentTopic.checkDeduplicationSnapshot(); + } + private void waitCacheInit(String topicName) throws Exception { pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe().close(); TopicName topic = TopicName.get(topicName); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 16a56af203973..577b9d5a6488a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -39,6 +39,8 @@ import com.google.common.collect.Sets; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; import io.netty.util.Timeout; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -79,6 +81,8 @@ import org.apache.bookkeeper.mledger.impl.cache.EntryCache; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.pulsar.broker.BrokerTestUtil; +import org.apache.pulsar.broker.service.ServerCnx; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.schema.GenericRecord; @@ -4474,4 +4478,50 @@ public void testListenerOrdering() throws Exception { assertEquals(values.get(i), "msg-" + i); } } + + @Test + public void testConsumeWhenDeliveryFailedByIOException() throws Exception { + final String topic = BrokerTestUtil.newUniqueName("persistent://my-property/my-ns/tp_"); + final String subscriptionName = "subscription1"; + final int messagesCount = 100; + final int receiverQueueSize = 1; + Producer producer = pulsarClient.newProducer(Schema.STRING).topic(topic).enableBatching(false).create(); + ConsumerImpl consumer = (ConsumerImpl) pulsarClient.newConsumer(Schema.STRING).topic(topic) + .subscriptionName(subscriptionName).receiverQueueSize(receiverQueueSize).subscribe(); + for (int i = 0; i < messagesCount; i++) { + producer.send(i + ""); + } + // Wait incoming queue of the consumer is full. + Awaitility.await().untilAsserted(() -> { + assertEquals(consumer.getIncomingMessageSize(), receiverQueueSize); + }); + + // Mock an io error for sending messages out. + ServerCnx serverCnx = (ServerCnx) pulsar.getBrokerService().getTopic(topic, false).join().get() + .getSubscription(subscriptionName).getDispatcher().getConsumers().iterator().next().cnx(); + serverCnx.ctx().channel().pipeline().addFirst(new ChannelDuplexHandler() { + + @Override + public void flush(ChannelHandlerContext ctx) throws Exception { + throw new IOException("Mocked error"); + } + }); + + // Verify all messages will be consumed. + Set receivedMessages = new HashSet<>(); + while (true) { + Message msg = consumer.receive(2, TimeUnit.SECONDS); + if (msg != null) { + receivedMessages.add(msg.getValue()); + consumer.acknowledge(msg); + } else { + break; + } + } + Assert.assertEquals(receivedMessages.size(), messagesCount); + + producer.close(); + consumer.close(); + admin.topics().delete(topic, false); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java new file mode 100644 index 0000000000000..5068a8ccb6b0b --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/HierarchyTopicAutoCreationTest.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import lombok.Cleanup; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import java.util.List; +import java.util.UUID; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.common.naming.TopicName; +import org.apache.pulsar.common.policies.data.AutoTopicCreationOverride; +import org.apache.pulsar.common.policies.data.Policies; +import org.apache.pulsar.metadata.api.MetadataCache; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test(groups = "broker-impl") +@Slf4j +public class HierarchyTopicAutoCreationTest extends ProducerConsumerBase { + + @Override + @BeforeMethod + protected void setup() throws Exception { + super.internalSetup(); + super.producerBaseSetup(); + } + + @Override + @AfterMethod(alwaysRun = true) + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test(invocationCount = 3) + @SneakyThrows + public void testPartitionedTopicAutoCreation() { + // Create namespace + final String namespace = "public/testPartitionedTopicAutoCreation"; + admin.namespaces().createNamespace(namespace); + // Set policies + final AutoTopicCreationOverride expectedPolicies = AutoTopicCreationOverride.builder() + .allowAutoTopicCreation(true) + .topicType("partitioned") + .defaultNumPartitions(1) + .build(); + admin.namespaces().setAutoTopicCreation(namespace, expectedPolicies); + // Double-check the policies + final AutoTopicCreationOverride nsAutoTopicCreationOverride = admin.namespaces() + .getAutoTopicCreation(namespace); + Assert.assertEquals(nsAutoTopicCreationOverride, expectedPolicies); + // Background invalidate cache + final MetadataCache nsCache = pulsar.getPulsarResources().getNamespaceResources().getCache(); + final Thread t1 = new Thread(() -> { + while (true) { + nsCache.invalidate("/admin/policies/" + namespace); + } + }); + t1.start(); + + // trigger auto-creation + final String topicName = "persistent://" + namespace + "/test-" + UUID.randomUUID(); + @Cleanup final Producer producer = pulsarClient.newProducer() + .topic(topicName) + .create(); + final List topics = admin.topics().getList(namespace); + Assert.assertEquals(topics.size(), 1); // expect only one topic + Assert.assertEquals(topics.get(0), + TopicName.get(topicName).getPartition(0).toString()); // expect partitioned topic + + // double-check policies + final AutoTopicCreationOverride actualPolicies2 = admin.namespaces().getAutoTopicCreation(namespace); + Assert.assertEquals(actualPolicies2, expectedPolicies); + + t1.interrupt(); + } +} \ No newline at end of file diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java new file mode 100644 index 0000000000000..692963884754e --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingDeduplicationTest.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import lombok.Cleanup; +import lombok.extern.slf4j.Slf4j; +import org.apache.pulsar.broker.service.Topic; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.ProducerConsumerBase; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.api.proto.MessageMetadata; +import org.apache.pulsar.common.protocol.Commands; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +@Test(groups = "broker-impl") +public class MessageChunkingDeduplicationTest extends ProducerConsumerBase { + + @BeforeClass + @Override + protected void setup() throws Exception { + this.conf.setBrokerDeduplicationEnabled(true); + super.internalSetup(); + super.producerBaseSetup(); + } + + @AfterClass(alwaysRun = true) + @Override + protected void cleanup() throws Exception { + super.internalCleanup(); + } + + @Test + public void testSendChunkMessageWithSameSequenceID() throws Exception { + String topicName = "persistent://my-property/my-ns/testSendChunkMessageWithSameSequenceID"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + @Cleanup + Producer producer = pulsarClient + .newProducer(Schema.STRING) + .producerName(producerName) + .topic(topicName) + .enableChunking(true) + .enableBatching(false) + .create(); + int messageSize = 6000; // payload size in KB + String message = String.join("", Collections.nCopies(messageSize * 1000, "a")); + producer.newMessage().value(message).sequenceId(10).send(); + Message msg = consumer.receive(10, TimeUnit.SECONDS); + assertNotNull(msg); + assertTrue(msg.getMessageId() instanceof ChunkMessageIdImpl); + assertEquals(msg.getValue(), message); + producer.newMessage().value(message).sequenceId(10).send(); + msg = consumer.receive(3, TimeUnit.SECONDS); + assertNull(msg); + } + + @Test + public void testDeduplicateChunksInSingleChunkMessages() throws Exception { + String topicName = "persistent://my-property/my-ns/testDeduplicateChunksInSingleChunkMessage"; + String producerName = "test-producer"; + @Cleanup + Consumer consumer = pulsarClient + .newConsumer(Schema.STRING) + .subscriptionName("test-sub") + .topic(topicName) + .subscribe(); + final PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService() + .getTopicIfExists(topicName).get().orElse(null); + assertNotNull(persistentTopic); + sendChunk(persistentTopic, producerName, 1, 0, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + sendChunk(persistentTopic, producerName, 1, 1, 2); + + Message message = consumer.receive(15, TimeUnit.SECONDS); + assertEquals(message.getData().length, 2); + + sendChunk(persistentTopic, producerName, 2, 0, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 1, 3); + sendChunk(persistentTopic, producerName, 2, 2, 3); + message = consumer.receive(20, TimeUnit.SECONDS); + assertEquals(message.getData().length, 3); + } + + protected static void sendChunk(final PersistentTopic persistentTopic, + final String producerName, + final long sequenceId, + final Integer chunkId, + final Integer numChunks) { + final MessageMetadata metadata = new MessageMetadata(); + metadata.setProducerName(producerName); + metadata.setSequenceId(sequenceId); + metadata.setPublishTime(System.currentTimeMillis()); + if (chunkId != null && numChunks != null) { + metadata.setUuid(producerName + "-" + sequenceId); + metadata.setChunkId(chunkId); + metadata.setNumChunksFromMsg(numChunks); + metadata.setTotalChunkMsgSize(numChunks); + } + final ByteBuf buf = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, + Unpooled.wrappedBuffer("a".getBytes())); + persistentTopic.publishMessage(buf, new Topic.PublishContext() { + @Override + public boolean isChunked() { + return chunkId != null; + } + + @Override + public String getProducerName() { + return producerName; + } + + public long getSequenceId() { + return sequenceId; + } + + @Override + public void completed(Exception e, long ledgerId, long entryId) { + String name = producerName + "-" + sequenceId; + if (chunkId != null) { + name += "-" + chunkId + "-" + numChunks; + } + if (e == null) { + log.info("Sent {} to ({}, {})", name, ledgerId, entryId); + } else { + log.error("Failed to send {}: {}", name, e.getMessage()); + } + } + }); + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java index c223208a8ef34..8016928b71201 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageChunkingTest.java @@ -53,6 +53,7 @@ import org.apache.pulsar.client.api.ProducerConsumerBase; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SizeUnit; import org.apache.pulsar.client.impl.MessageImpl.SchemaState; import org.apache.pulsar.client.impl.ProducerImpl.OpSendMsg; @@ -332,6 +333,78 @@ public void testMaxPendingChunkMessages() throws Exception { } + @Test + public void testResendChunkMessagesWithoutAckHole() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessagesWithoutAckHole"; + final String subName = "my-subscriber-name"; + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName(subName) + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + assertEquals(admin.topics().getStats(topicName).getSubscriptions().get(subName) + .getNonContiguousDeletedMessagesRanges(), 0); + } + + @Test + public void testResendChunkMessages() throws Exception { + log.info("-- Starting {} test --", methodName); + final String topicName = "persistent://my-property/my-ns/testResendChunkMessages"; + + @Cleanup + Consumer consumer = pulsarClient.newConsumer(Schema.STRING) + .topic(topicName) + .subscriptionName("my-subscriber-name") + .maxPendingChunkedMessage(10) + .autoAckOldestChunkedMessageOnQueueFull(true) + .subscribe(); + @Cleanup + Producer producer = pulsarClient.newProducer(Schema.STRING) + .topic(topicName) + .enableChunking(true) + .enableBatching(false) + .create(); + + sendSingleChunk(producer, "0", 0, 2); + + sendSingleChunk(producer, "0", 0, 2); // Resending the first chunk + sendSingleChunk(producer, "1", 0, 3); // This is for testing the interwoven chunked message + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 0, 3); // Resending the UUID-1 chunked message + + sendSingleChunk(producer, "0", 1, 2); + + Message receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-0-0|chunk-0-1|"); + consumer.acknowledge(receivedMsg); + + sendSingleChunk(producer, "1", 1, 3); + sendSingleChunk(producer, "1", 2, 3); + + receivedMsg = consumer.receive(5, TimeUnit.SECONDS); + assertEquals(receivedMsg.getValue(), "chunk-1-0|chunk-1-1|chunk-1-2|"); + consumer.acknowledge(receivedMsg); + Assert.assertEquals(((ConsumerImpl) consumer).getAvailablePermits(), 8); + } + /** * Validate that chunking is not supported with batching and non-persistent topic * @@ -514,4 +587,17 @@ private String createMessagePayload(int size) { return str.toString(); } + private void sendSingleChunk(Producer producer, String uuid, int chunkId, int totalChunks) + throws PulsarClientException { + TypedMessageBuilderImpl msg = (TypedMessageBuilderImpl) producer.newMessage() + .value(String.format("chunk-%s-%d|", uuid, chunkId)); + MessageMetadata msgMetadata = msg.getMetadataBuilder(); + msgMetadata.setUuid(uuid) + .setChunkId(chunkId) + .setNumChunksFromMsg(totalChunks) + .setTotalChunkMsgSize(100); + msg.send(); + } + + } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java index ddad53fbc829f..0eec3e31cca1d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactionTest.java @@ -22,12 +22,13 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; -import static org.testng.Assert.assertFalse; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.netty.buffer.ByteBuf; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -45,9 +46,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import io.netty.buffer.ByteBuf; import lombok.Cleanup; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.api.OpenBuilder; import org.apache.bookkeeper.mledger.AsyncCallbacks; @@ -61,6 +62,7 @@ import org.apache.pulsar.broker.service.Topic; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.broker.service.persistent.SystemTopic; +import org.apache.pulsar.client.admin.LongRunningProcessStatus; import org.apache.pulsar.client.api.CompressionType; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.CryptoKeyReader; @@ -70,11 +72,14 @@ import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.ProducerBuilder; +import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.Reader; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionInitialPosition; import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.BatchMessageIdImpl; +import org.apache.pulsar.client.impl.ConsumerImpl; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.policies.data.ClusterData; @@ -91,6 +96,7 @@ import org.testng.annotations.Test; @Test(groups = "flaky") +@Slf4j public class CompactionTest extends MockedPulsarServiceBaseTest { private ScheduledExecutorService compactionScheduler; private BookKeeper bk; @@ -530,6 +536,61 @@ public void testBatchMessageIdsDontChange() throws Exception { } } + @Test + public void testBatchMessageWithNullValue() throws Exception { + String topic = "persistent://my-property/use/my-ns/my-topic1"; + + pulsarClient.newConsumer().topic(topic).subscriptionName("sub1") + .receiverQueueSize(1).readCompacted(true).subscribe().close(); + + try (Producer producer = pulsarClient.newProducer().topic(topic) + .maxPendingMessages(3) + .enableBatching(true) + .batchingMaxMessages(3) + .batchingMaxPublishDelay(1, TimeUnit.HOURS) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create() + ) { + // batch 1 + producer.newMessage().key("key1").value("my-message-1".getBytes()).sendAsync(); + producer.newMessage().key("key1").value(null).sendAsync(); + producer.newMessage().key("key2").value("my-message-3".getBytes()).send(); + + // batch 2 + producer.newMessage().key("key3").value("my-message-4".getBytes()).sendAsync(); + producer.newMessage().key("key3").value("my-message-5".getBytes()).sendAsync(); + producer.newMessage().key("key3").value("my-message-6".getBytes()).send(); + + // batch 3 + producer.newMessage().key("key4").value("my-message-7".getBytes()).sendAsync(); + producer.newMessage().key("key4").value(null).sendAsync(); + producer.newMessage().key("key5").value("my-message-9".getBytes()).send(); + } + + + // compact the topic + Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + compactor.compact(topic); + + // Read messages before compaction to get ids + List> messages = new ArrayList<>(); + try (Consumer consumer = pulsarClient.newConsumer().topic(topic) + .subscriptionName("sub1").receiverQueueSize(1).readCompacted(true).subscribe()) { + while (true) { + Message message = consumer.receive(5, TimeUnit.SECONDS); + if (message == null) { + break; + } + messages.add(message); + } + } + + assertEquals(messages.size(), 3); + assertEquals(messages.get(0).getKey(), "key2"); + assertEquals(messages.get(1).getKey(), "key3"); + assertEquals(messages.get(2).getKey(), "key5"); + } + @Test public void testWholeBatchCompactedOut() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; @@ -1748,4 +1809,38 @@ public void addFailed(ManagedLedgerException exception, Object ctx) { Assert.assertNotEquals(ledgerId, -1L); }); } + + @Test(timeOut = 100000) + public void testReceiverQueueSize() throws Exception { + final String topicName = "persistent://my-property/use/my-ns/testReceiverQueueSize" + UUID.randomUUID(); + final String subName = "my-sub"; + final int receiveQueueSize = 1; + @Cleanup + PulsarClient client = newPulsarClient(lookupUrl.toString(), 100); + Producer producer = pulsarClient.newProducer(Schema.STRING) + .enableBatching(false).topic(topicName).create(); + + for (int i = 0; i < 10; i++) { + producer.newMessage().key(String.valueOf(i % 2)).value(String.valueOf(i)).sendAsync(); + } + producer.flush(); + + admin.topics().triggerCompaction(topicName); + + Awaitility.await().untilAsserted(() -> { + assertEquals(admin.topics().compactionStatus(topicName).status, + LongRunningProcessStatus.Status.SUCCESS); + }); + + ConsumerImpl consumer = (ConsumerImpl) client.newConsumer(Schema.STRING) + .topic(topicName).readCompacted(true).receiverQueueSize(receiveQueueSize).subscriptionName(subName) + .subscribe(); + + //Give some time to consume + Awaitility.await() + .untilAsserted(() -> Assert.assertEquals(consumer.getStats().getMsgNumInReceiverQueue().intValue(), + receiveQueueSize)); + consumer.close(); + producer.close(); + } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java index bd875384aecb3..304aae3f57f7b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorTest.java @@ -19,13 +19,10 @@ package org.apache.pulsar.compaction; import static org.apache.pulsar.client.impl.RawReaderTest.extractKey; - import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.netty.buffer.ByteBuf; - import java.util.ArrayList; import java.util.Enumeration; import java.util.HashMap; @@ -33,22 +30,29 @@ import java.util.Map; import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; - +import lombok.Cleanup; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; +import org.apache.bookkeeper.mledger.Entry; +import org.apache.bookkeeper.mledger.Position; +import org.apache.bookkeeper.mledger.impl.PositionImpl; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; +import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.RawMessage; import org.apache.pulsar.client.impl.PulsarClientImpl; import org.apache.pulsar.client.impl.RawMessageImpl; import org.apache.pulsar.common.policies.data.ClusterData; -import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.protocol.Commands; +import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -59,7 +63,6 @@ public class CompactorTest extends MockedPulsarServiceBaseTest { private ScheduledExecutorService compactionScheduler; - @BeforeMethod @Override public void setup() throws Exception { @@ -82,18 +85,19 @@ public void cleanup() throws Exception { compactionScheduler.shutdownNow(); } - private List compactAndVerify(String topic, Map expected, boolean checkMetrics) throws Exception { + private List compactAndVerify(String topic, Map expected, boolean checkMetrics) + throws Exception { BookKeeper bk = pulsar.getBookKeeperClientFactory().create( this.conf, null, null, Optional.empty(), null); Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); long compactedLedgerId = compactor.compact(topic).get(); LedgerHandle ledger = bk.openLedger(compactedLedgerId, - Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, - Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); + Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, + Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD); Assert.assertEquals(ledger.getLastAddConfirmed() + 1, // 0..lac - expected.size(), - "Should have as many entries as there is keys"); + expected.size(), + "Should have as many entries as there is keys"); List keys = new ArrayList<>(); Enumeration entries = ledger.readEntries(0, ledger.getLastAddConfirmed()); @@ -107,7 +111,7 @@ private List compactAndVerify(String topic, Map expected byte[] bytes = new byte[payload.readableBytes()]; payload.readBytes(bytes); Assert.assertEquals(bytes, expected.remove(key), - "Compacted version should match expected version"); + "Compacted version should match expected version"); m.close(); } if (checkMetrics) { @@ -131,17 +135,18 @@ public void testCompaction() throws Exception { final int numMessages = 1000; final int maxKeys = 10; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); Random r = new Random(0); for (int j = 0; j < numMessages; j++) { int keyIndex = r.nextInt(maxKeys); - String key = "key"+keyIndex; + String key = "key" + keyIndex; byte[] data = ("my-message-" + key + "-" + j).getBytes(); producer.newMessage() .key(key) @@ -156,10 +161,11 @@ public void testCompaction() throws Exception { public void testCompactAddCompact() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); Map expected = new HashMap<>(); @@ -193,10 +199,11 @@ public void testCompactAddCompact() throws Exception { public void testCompactedInOrder() throws Exception { String topic = "persistent://my-property/use/my-ns/my-topic1"; + @Cleanup Producer producer = pulsarClient.newProducer().topic(topic) - .enableBatching(false) - .messageRoutingMode(MessageRoutingMode.SinglePartition) - .create(); + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); producer.newMessage() .key("c") @@ -243,6 +250,51 @@ public void testPhaseOneLoopTimeConfiguration() { } + @Test + public void testCompactedWithConcurrentSend() throws Exception { + String topic = "persistent://my-property/use/my-ns/testCompactedWithConcurrentSend"; + + @Cleanup + Producer producer = pulsarClient.newProducer().topic(topic) + .enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition) + .create(); + + + BookKeeper bk = pulsar.getBookKeeperClientFactory().create( + this.conf, null, null, Optional.empty(), null); + Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler); + + CompletableFuture future = CompletableFuture.runAsync(() -> { + for (int i = 0; i < 100; i++) { + try { + producer.newMessage().key(String.valueOf(i)).value(String.valueOf(i).getBytes()).send(); + } catch (PulsarClientException e) { + throw new RuntimeException(e); + } + } + }); + + PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get(); + CompactedTopic compactedTopic = persistentTopic.getCompactedTopic(); + + Awaitility.await().untilAsserted(() -> { + long compactedLedgerId = compactor.compact(topic).get(); + Thread.sleep(300); + Optional compactedTopicContext = persistentTopic.getCompactedTopicContext(); + Assert.assertTrue(compactedTopicContext.isPresent()); + Assert.assertEquals(compactedTopicContext.get().ledger.getId(), compactedLedgerId); + }); + + Position lastCompactedPosition = compactedTopic.getCompactionHorizon().get(); + Entry lastCompactedEntry = compactedTopic.readLastEntryOfCompactedLedger().get(); + + Assert.assertTrue(PositionImpl.get(lastCompactedPosition.getLedgerId(), lastCompactedPosition.getEntryId()) + .compareTo(PositionImpl.get(lastCompactedEntry.getLedgerId(), lastCompactedEntry.getEntryId())) >= 0); + + future.join(); + } + public ByteBuf extractPayload(RawMessage m) throws Exception { ByteBuf payloadAndMetadata = m.getHeadersAndPayload(); Commands.skipChecksumIfPresent(payloadAndMetadata); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java index 7133684721c9b..dcd6671780a6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/schema/SchemaTest.java @@ -1179,6 +1179,33 @@ private void testIncompatibleSchema() throws Exception { } } + /** + * This test just ensure that schema check still keeps the original logic: if there has any producer, but no schema + * was registered, the new consumer could not register new schema. + * TODO: I think this design should be improved: if a producer used "AUTO_PRODUCE_BYTES" schema, we should allow + * the new consumer to register new schema. But before we can solve this problem, we need to modify + * "CmdProducer" to let the Broker know that the Producer uses a schema of type "AUTO_PRODUCE_BYTES". + */ + @Test + public void testAutoProduceAndSpecifiedConsumer() throws Exception { + final String namespace = PUBLIC_TENANT + "/ns_" + randomName(16); + admin.namespaces().createNamespace(namespace, Sets.newHashSet(CLUSTER_NAME)); + final String topicName = "persistent://" + namespace + "/tp_" + randomName(16); + admin.topics().createNonPartitionedTopic(topicName); + + Producer producer = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()).topic(topicName).create(); + try { + pulsarClient.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub1").subscribe(); + fail("Should throw ex: Topic does not have schema to check"); + } catch (Exception ex){ + assertTrue(ex.getMessage().contains("Topic does not have schema to check")); + } + + // Cleanup. + producer.close(); + admin.topics().delete(topicName); + } + @Test public void testCreateSchemaInParallel() throws Exception { final String namespace = "test-namespace-" + randomName(16); diff --git a/pulsar-client-1x-base/pom.xml b/pulsar-client-1x-base/pom.xml index 852d4af907490..bd90997f2b18f 100644 --- a/pulsar-client-1x-base/pom.xml +++ b/pulsar-client-1x-base/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-1x-base Pulsar Client 1.x Compatibility Base pom - pulsar-client-2x-shaded pulsar-client-1x - @@ -54,7 +50,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -70,5 +65,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-1x/pom.xml b/pulsar-client-1x-base/pulsar-client-1x/pom.xml index 311dc3445d987..8839ace7e1835 100644 --- a/pulsar-client-1x-base/pulsar-client-1x/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-1x/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-1x Pulsar Client 1.x Compatibility API - ${project.groupId} pulsar-client-2x-shaded ${project.version} - com.google.guava guava - org.apache.commons commons-lang3 - - @@ -70,7 +63,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -87,5 +79,4 @@ - diff --git a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml index 8b0f3f517bc7a..02011e78f5aaa 100644 --- a/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml +++ b/pulsar-client-1x-base/pulsar-client-2x-shaded/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-client-1x-base - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-2x-shaded Pulsar Client 2.x Shaded API - ${project.groupId} @@ -39,10 +36,9 @@ ${project.version} - - + org.apache.maven.plugins maven-shade-plugin @@ -56,16 +52,15 @@ true true false - - org.apache.pulsar:pulsar-client - org.apache.pulsar:pulsar-client-api + io.streamnative:pulsar-client + io.streamnative:pulsar-client-api - org.apache.pulsar:pulsar-client + io.streamnative:pulsar-client ** @@ -93,6 +88,30 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin-api/pom.xml b/pulsar-client-admin-api/pom.xml index e06d71e039c34..938db21e9002c 100644 --- a/pulsar-client-admin-api/pom.xml +++ b/pulsar-client-admin-api/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar - 2.10.5-SNAPSHOT - .. - - - pulsar-client-admin-api - Pulsar Client Admin :: API - - - - ${project.groupId} - pulsar-client-api - ${project.version} - - - - org.slf4j - slf4j-api - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 2.10.5.3 + .. + + pulsar-client-admin-api + Pulsar Client Admin :: API + + + ${project.groupId} + pulsar-client-api + ${project.version} + + + org.slf4j + slf4j-api + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-admin-shaded/pom.xml b/pulsar-client-admin-shaded/pom.xml index 1e54993be9118..da22cdd3634dc 100644 --- a/pulsar-client-admin-shaded/pom.xml +++ b/pulsar-client-admin-shaded/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-admin Pulsar Client Admin - ${project.groupId} @@ -74,7 +70,6 @@ - maven-antrun-plugin @@ -87,15 +82,12 @@ - + - org.apache.maven.plugins maven-shade-plugin @@ -108,11 +100,10 @@ true true - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -125,7 +116,7 @@ com.google.code.gson:gson com.fasterxml.jackson.core io.netty:* - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -147,12 +138,12 @@ org.yaml:snakeyaml io.swagger:* - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -162,7 +153,7 @@ - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-admin-original ** @@ -173,7 +164,7 @@ - + org.asynchttpclient org.apache.pulsar.shade.org.asynchttpclient @@ -248,51 +239,75 @@ org.reactivestreams org.apache.pulsar.shade.org.reactivestreams - - io.grpc - org.apache.pulsar.shade.io.grpc - - - okio - org.apache.pulsar.shade.okio - - - com.squareup - org.apache.pulsar.shade.com.squareup - - - io.opencensus - org.apache.pulsar.shade.io.opencensus - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - org.objenesis - org.apache.pulsar.shade.org.objenesis - - - org.yaml - org.apache.pulsar.shade.org.yaml - - - io.swagger - org.apache.pulsar.shade.io.swagger - - - org.apache.bookkeeper - org.apache.pulsar.shade.org.apache.bookkeeper - + + io.grpc + org.apache.pulsar.shade.io.grpc + + + okio + org.apache.pulsar.shade.okio + + + com.squareup + org.apache.pulsar.shade.com.squareup + + + io.opencensus + org.apache.pulsar.shade.io.opencensus + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + org.objenesis + org.apache.pulsar.shade.org.objenesis + + + org.yaml + org.apache.pulsar.shade.org.yaml + + + io.swagger + org.apache.pulsar.shade.io.swagger + + + org.apache.bookkeeper + org.apache.pulsar.shade.org.apache.bookkeeper + - - + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + diff --git a/pulsar-client-admin/pom.xml b/pulsar-client-admin/pom.xml index e2fe7cc91165c..6f22b0b7a6f2c 100644 --- a/pulsar-client-admin/pom.xml +++ b/pulsar-client-admin/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-admin-original Pulsar Client Admin Original - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - org.glassfish.jersey.core jersey-client - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.media jersey-media-multipart - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api @@ -91,29 +79,24 @@ javax.activation runtime - org.slf4j jul-to-slf4j - com.google.guava guava - com.google.code.gson gson - ${project.groupId} pulsar-package-core ${project.version} - @@ -133,7 +116,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-all/pom.xml b/pulsar-client-all/pom.xml index 062f9629f77d2..98fa3e34904ba 100644 --- a/pulsar-client-all/pom.xml +++ b/pulsar-client-all/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-all Pulsar Client All - ${project.groupId} @@ -55,10 +52,8 @@ true - - org.apache.maven.plugins maven-dependency-plugin @@ -94,7 +89,6 @@ - maven-antrun-plugin @@ -107,15 +101,12 @@ - + - org.apache.maven.plugins @@ -130,12 +121,11 @@ true true false - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-transaction-common - org.apache.pulsar:pulsar-client-admin-original + io.streamnative:pulsar-client-original + io.streamnative:pulsar-transaction-common + io.streamnative:pulsar-client-admin-original org.apache.commons:commons-lang3 commons-codec:commons-codec commons-collections:commons-collections @@ -164,8 +154,7 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common org.apache.bookkeeper:* com.yahoo.datasketches:sketches-core org.glassfish.jersey*:* @@ -192,12 +181,12 @@ org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -292,24 +281,24 @@ org.apache.pulsar.shade.org.glassfish - io.grpc - org.apache.pulsar.shade.io.grpc + io.grpc + org.apache.pulsar.shade.io.grpc - okio - org.apache.pulsar.shade.okio + okio + org.apache.pulsar.shade.okio - com.squareup - org.apache.pulsar.shade.com.squareup + com.squareup + org.apache.pulsar.shade.com.squareup - io.opencensus - org.apache.pulsar.shade.io.opencensus + io.opencensus + org.apache.pulsar.shade.io.opencensus - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty org.objenesis @@ -366,14 +355,13 @@ - - + + - - - 4.0.0 - - - org.apache.pulsar - pulsar - 2.10.5-SNAPSHOT - .. - - - pulsar-client-api - Pulsar Client :: API - - - - - - com.google.protobuf - protobuf-java - provided - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - + + com.google.protobuf + protobuf-java + provided + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-client-api/src/main/resources/findbugsExclude.xml b/pulsar-client-api/src/main/resources/findbugsExclude.xml index 9d73ac29a7bdb..353f01a7bb285 100644 --- a/pulsar-client-api/src/main/resources/findbugsExclude.xml +++ b/pulsar-client-api/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-client-auth-athenz/pom.xml b/pulsar-client-auth-athenz/pom.xml index d0f032db40c13..9abeb20d76dec 100644 --- a/pulsar-client-auth-athenz/pom.xml +++ b/pulsar-client-auth-athenz/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-auth-athenz jar Athenz authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.yahoo.athenz athenz-zts-java-client-core - com.google.guava guava - org.apache.commons commons-lang3 - @@ -77,7 +69,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -95,7 +86,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-auth-athenz/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-auth-sasl jar SASL authentication plugin for java client - - ${project.groupId} pulsar-client-original ${project.parent.version} true - com.google.guava guava - org.apache.commons commons-lang3 - org.projectlombok lombok - javax.ws.rs javax.ws.rs-api - org.glassfish.jersey.core jersey-client - - @@ -88,7 +77,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-client-cpp/eclipse-formatter.xml b/pulsar-client-cpp/eclipse-formatter.xml index 12c2a2171bcac..41f617cc914e9 100644 --- a/pulsar-client-cpp/eclipse-formatter.xml +++ b/pulsar-client-cpp/eclipse-formatter.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-client-messagecrypto-bc/pom.xml b/pulsar-client-messagecrypto-bc/pom.xml index 74f14e406e8ff..e80294fc02c9f 100644 --- a/pulsar-client-messagecrypto-bc/pom.xml +++ b/pulsar-client-messagecrypto-bc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-messagecrypto-bc jar Message crypto for End to End encryption with BouncyCastleProvider - ${project.groupId} @@ -40,7 +37,6 @@ ${project.parent.version} provided - ${project.groupId} bouncy-castle-bc @@ -48,9 +44,7 @@ pkg true - - diff --git a/pulsar-client-shaded/pom.xml b/pulsar-client-shaded/pom.xml index e28a03e99ab3a..4dc1a4518f3b5 100644 --- a/pulsar-client-shaded/pom.xml +++ b/pulsar-client-shaded/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client Pulsar Client Java - ${project.groupId} @@ -49,11 +46,8 @@ ${project.version} - - - org.apache.maven.plugins maven-dependency-plugin @@ -89,7 +83,6 @@ - maven-antrun-plugin @@ -102,15 +95,12 @@ - + - org.apache.maven.plugins @@ -125,11 +115,10 @@ true true false - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-transaction-common + io.streamnative:pulsar-client-original + io.streamnative:pulsar-transaction-common org.apache.bookkeeper:* org.apache.commons:commons-lang3 commons-codec:commons-codec @@ -157,24 +146,22 @@ commons-*:* io.swagger:* io.airlift:* - - org.apache.pulsar:pulsar-common + io.streamnative:pulsar-common com.yahoo.datasketches:sketches-core org.objenesis:* org.yaml:snakeyaml - org.apache.avro:* com.thoughtworks.paranamer:paranamer org.apache.commons:commons-compress org.tukaani:xz - org.apache.pulsar:pulsar-client-messagecrypto-bc + io.streamnative:pulsar-client-messagecrypto-bc - org.apache.pulsar:pulsar-client-original + io.streamnative:pulsar-client-original ** @@ -299,14 +286,13 @@ - - + + - com.github.spotbugs spotbugs-maven-plugin @@ -320,7 +306,6 @@ - - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-tools-test Pulsar Client Tools Test Pulsar Client Tools Test - ${project.groupId} @@ -82,7 +80,6 @@ - org.apache.maven.plugins maven-deploy-plugin diff --git a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml +++ b/pulsar-client-tools-test/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-tools Pulsar Client Tools Pulsar Client Tools - com.beust @@ -85,7 +83,6 @@ pulsar-websocket ${project.version} - ${project.groupId} @@ -93,15 +90,13 @@ ${project.version} test - - org.apache.pulsar - pulsar-io-batch-discovery-triggerers - ${project.version} - test + io.streamnative + pulsar-io-batch-discovery-triggerers + ${project.version} + test - @@ -121,7 +116,6 @@ - @@ -141,7 +135,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -157,5 +150,4 @@ - diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index 96a259371a648..e1876d1df68cc 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-client-original Pulsar Client Java - ${project.groupId} pulsar-client-api ${project.parent.version} - ${project.groupId} pulsar-common ${project.parent.version} - ${project.groupId} pulsar-transaction-common ${project.parent.version} - ${project.groupId} bouncy-castle-bc ${project.parent.version} pkg - ${project.groupId} @@ -65,7 +58,6 @@ ${project.parent.version} true - io.netty netty-codec-http @@ -78,55 +70,44 @@ io.netty netty-codec-socks - io.netty netty-resolver-dns - org.apache.commons commons-lang3 - org.asynchttpclient async-http-client - com.typesafe.netty netty-reactive-streams - org.slf4j slf4j-api - commons-codec commons-codec - com.yahoo.datasketches sketches-core - com.google.code.gson gson - - org.apache.avro avro ${avro.version} - org.apache.avro avro-protobuf @@ -138,36 +119,30 @@ - joda-time joda-time provided - com.google.protobuf protobuf-java provided - com.fasterxml.jackson.module jackson-module-jsonSchema - net.jcip jcip-annotations - com.github.spotbugs spotbugs-annotations provided true - ${project.groupId} @@ -175,22 +150,18 @@ ${project.parent.version} test - org.skyscreamer jsonassert ${skyscreamer.version} test - org.awaitility awaitility test - - @@ -217,7 +188,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin @@ -231,7 +201,6 @@ - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java index 9b4d1b7d683dd..f762b5ab45038 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/AbstractBatchMessageContainer.java @@ -35,7 +35,6 @@ public abstract class AbstractBatchMessageContainer implements BatchMessageConta protected CompressionType compressionType; protected CompressionCodec compressor; protected String topicName; - protected String producerName; protected ProducerImpl producer; protected int maxNumMessagesInBatch; @@ -98,7 +97,6 @@ public ProducerImpl.OpSendMsg createOpSendMsg() throws IOException { public void setProducer(ProducerImpl producer) { this.producer = producer; this.topicName = producer.getTopic(); - this.producerName = producer.getProducerName(); this.compressionType = CompressionCodecProvider .convertToWireProtocol(producer.getConfiguration().getCompressionType()); this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java index 37b522d8902fa..f657816b6dd04 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageContainerImpl.java @@ -80,8 +80,8 @@ public BatchMessageContainerImpl(ProducerImpl producer) { public boolean add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, - numMessagesInBatch); + log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, + producer.getProducerName(), numMessagesInBatch); } if (++numMessagesInBatch == 1) { @@ -193,8 +193,8 @@ public void discard(Exception ex) { firstCallback.sendComplete(ex); } } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, producerName, - lowestSequenceId, t); + log.warn("[{}] [{}] Got exception while completing the callback for msg {}:", topicName, + producer.getProducerName(), lowestSequenceId, t); } clear(); } @@ -226,6 +226,14 @@ public OpSendMsg createOpSendMsg() throws IOException { ByteBufPair cmd = producer.sendMessage(producer.producerId, messageMetadata.getSequenceId(), messageMetadata.getHighestSequenceId(), numMessagesInBatch, messageMetadata, encryptedPayload); + if (log.isDebugEnabled()) { + log.debug("[{}] [{}] Build batch msg seq:{}, highest-seq:{}, numMessagesInBatch: {}, uncompressedSize: {}," + + " payloadSize: {}", topicName, producer.getProducerName(), + messageMetadata.getSequenceId(), messageMetadata.getNumMessagesInBatch(), + messageMetadata.getHighestSequenceId(), + messageMetadata.getUncompressedSize(), encryptedPayload.readableBytes()); + } + OpSendMsg op = OpSendMsg.create(messages, cmd, messageMetadata.getSequenceId(), messageMetadata.getHighestSequenceId(), firstCallback); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java index 7614728353f89..9fdd44715f669 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageKeyBasedContainer.java @@ -54,8 +54,8 @@ class BatchMessageKeyBasedContainer extends AbstractBatchMessageContainer { @Override public boolean add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { - log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, producerName, - numMessagesInBatch); + log.debug("[{}] [{}] add message to batch, num messages in batch so far is {}", topicName, + producer.getProducerName(), numMessagesInBatch); } numMessagesInBatch++; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); @@ -68,7 +68,7 @@ public boolean add(MessageImpl msg, SendCallback callback) { part.compressor = compressor; part.maxBatchSize = maxBatchSize; part.topicName = topicName; - part.producerName = producerName; + part.producerName = producer.getProducerName(); batches.putIfAbsent(key, part); if (msg.getMessageBuilder().hasTxnidMostBits() && currentTxnidMostBits == -1) { @@ -104,7 +104,7 @@ public void discard(Exception ex) { // Need to protect ourselves from any exception being thrown in the future handler from the application batches.forEach((k, v) -> v.firstCallback.sendComplete(ex)); } catch (Throwable t) { - log.warn("[{}] [{}] Got exception while completing the callback", topicName, producerName, t); + log.warn("[{}] [{}] Got exception while completing the callback", topicName, producer.getProducerName(), t); } batches.forEach((k, v) -> ReferenceCountUtil.safeRelease(v.batchedMessageMetadataAndPayload)); clear(); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 4a7fef11af49b..e86651523167d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -34,6 +34,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -1416,7 +1418,9 @@ void messageReceived(CommandMessage cmdMessage, ByteBuf headersAndPayload, Clien private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata msgMetadata, MessageIdImpl msgId, MessageIdData messageId, ClientCnx cnx) { - + if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { + increaseAvailablePermits(cnx); + } // Lazy task scheduling to expire incomplete chunk message if (!expireChunkMessageTaskScheduled && expireTimeOfIncompleteChunkedMessageMillis > 0) { ((ScheduledExecutorService) client.getScheduledExecutorProvider().getExecutor()).scheduleAtFixedRate( @@ -1428,11 +1432,58 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m expireChunkMessageTaskScheduled = true; } + ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid()); + if (msgMetadata.getChunkId() == 0) { + if (chunkedMsgCtx != null) { + // Handle ack hole case when receive duplicated chunks. + // There are two situation that receives chunks with the same sequence ID and chunk ID. + // Situation 1 - Message redeliver: + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // In this case, chunk-3 and chunk-4 have the same msgID with chunk-1 and chunk-2. + // This may be caused by message redeliver, we can't ack any chunk in this case here. + // Situation 2 - Corrupted chunk message + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 0, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // In this case, all the chunks with different msgIDs and are persistent in the topic. + // But Chunk-1 and Chunk-2 belong to a corrupted chunk message that must be skipped since + // they will not be delivered to end users. So we should ack them here to avoid ack hole. + boolean isCorruptedChunkMessageDetected = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (isCorruptedChunkMessageDetected) { + Arrays.stream(chunkedMsgCtx.chunkedMessageIds).forEach(messageId1 -> { + if (messageId1 != null) { + doAcknowledge(messageId1, AckType.Individual, Collections.emptyMap(), null); + } + }); + } + // The first chunk of a new chunked-message received before receiving other chunks of previous + // chunked-message + // so, remove previous chunked-message from map and release buffer + if (chunkedMsgCtx.chunkedMsgBuffer != null) { + ReferenceCountUtil.safeRelease(chunkedMsgCtx.chunkedMsgBuffer); + } + chunkedMsgCtx.recycle(); + chunkedMessagesMap.remove(msgMetadata.getUuid()); + } + pendingChunkedMessageCount++; + if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) { + removeOldestPendingChunkedMessage(); + } + int totalChunks = msgMetadata.getNumChunksFromMsg(); ByteBuf chunkedMsgBuffer = PulsarByteBufAllocator.DEFAULT.buffer(msgMetadata.getTotalChunkMsgSize(), msgMetadata.getTotalChunkMsgSize()); - int totalChunks = msgMetadata.getNumChunksFromMsg(); - chunkedMessagesMap.computeIfAbsent(msgMetadata.getUuid(), + chunkedMsgCtx = chunkedMessagesMap.computeIfAbsent(msgMetadata.getUuid(), (key) -> ChunkedMessageCtx.get(totalChunks, chunkedMsgBuffer)); pendingChunkedMessageCount++; if (maxPendingChunkedMessage > 0 && pendingChunkedMessageCount > maxPendingChunkedMessage) { @@ -1441,12 +1492,37 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m pendingChunkedMessageUuidQueue.add(msgMetadata.getUuid()); } - ChunkedMessageCtx chunkedMsgCtx = chunkedMessagesMap.get(msgMetadata.getUuid()); // discard message if chunk is out-of-order if (chunkedMsgCtx == null || chunkedMsgCtx.chunkedMsgBuffer == null || msgMetadata.getChunkId() != (chunkedMsgCtx.lastChunkedMessageId + 1)) { + // Filter and ack duplicated chunks instead of discard ctx. + // For example: + // Chunk-1 sequence ID: 0, chunk ID: 0, msgID: 1:1 + // Chunk-2 sequence ID: 0, chunk ID: 1, msgID: 1:2 + // Chunk-3 sequence ID: 0, chunk ID: 2, msgID: 1:3 + // Chunk-4 sequence ID: 0, chunk ID: 1, msgID: 1:4 + // Chunk-5 sequence ID: 0, chunk ID: 2, msgID: 1:5 + // Chunk-6 sequence ID: 0, chunk ID: 3, msgID: 1:6 + // We should filter and ack chunk-4 and chunk-5. + if (chunkedMsgCtx != null && msgMetadata.getChunkId() <= chunkedMsgCtx.lastChunkedMessageId) { + log.warn("[{}] Receive a duplicated chunk message with messageId [{}], last-chunk-Id [{}], " + + "chunkId [{}], sequenceId [{}]", + msgMetadata.getProducerName(), msgId, chunkedMsgCtx.lastChunkedMessageId, + msgMetadata.getChunkId(), msgMetadata.getSequenceId()); + compressedPayload.release(); + // Just like the above logic of receiving the first chunk again. We only ack this chunk in the message + // duplication case. + boolean isDuplicatedChunk = Arrays.stream(chunkedMsgCtx.chunkedMessageIds) + .noneMatch(messageId1 -> messageId1 != null && messageId1.ledgerId == messageId.getLedgerId() + && messageId1.entryId == messageId.getEntryId()); + if (isDuplicatedChunk) { + doAcknowledge(msgId, AckType.Individual, Collections.emptyMap(), null); + } + return null; + } // means we lost the first chunk: should never happen - log.info("Received unexpected chunk messageId {}, last-chunk-id{}, chunkId = {}", msgId, + log.info("[{}] [{}] Received unexpected chunk messageId {}, last-chunk-id = {}, chunkId = {}", topic, + subscription, msgId, (chunkedMsgCtx != null ? chunkedMsgCtx.lastChunkedMessageId : null), msgMetadata.getChunkId()); if (chunkedMsgCtx != null) { if (chunkedMsgCtx.chunkedMsgBuffer != null) { @@ -1456,7 +1532,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m } chunkedMessagesMap.remove(msgMetadata.getUuid()); compressedPayload.release(); - increaseAvailablePermits(cnx); if (expireTimeOfIncompleteChunkedMessageMillis > 0 && System.currentTimeMillis() > (msgMetadata.getPublishTime() + expireTimeOfIncompleteChunkedMessageMillis)) { @@ -1475,7 +1550,6 @@ private ByteBuf processMessageChunk(ByteBuf compressedPayload, MessageMetadata m // if final chunk is not received yet then release payload and return if (msgMetadata.getChunkId() != (msgMetadata.getNumChunksFromMsg() - 1)) { compressedPayload.release(); - increaseAvailablePermits(cnx); return null; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java index 56443709bf354..954375919d80b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/TransactionMetaStoreHandler.java @@ -167,6 +167,8 @@ public CompletableFuture connectionOpened(ClientCnx cnx) { return null; }); } else { + LOG.warn("Can not connect to the transaction coordinator because the protocol version {} is " + + "lower than 19", cnx.getRemoteEndpointProtocolVersion()); registerToConnection(cnx); future.complete(null); } diff --git a/pulsar-client/src/main/resources/findbugsExclude.xml b/pulsar-client/src/main/resources/findbugsExclude.xml index a37c886466ea4..121204d24e564 100644 --- a/pulsar-client/src/main/resources/findbugsExclude.xml +++ b/pulsar-client/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index 08889451dec31..580146001b4f5 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-common Pulsar Common Common libraries needed by client/broker/tools - ${project.groupId} pulsar-client-api ${project.version} - ${project.groupId} pulsar-client-admin-api ${project.version} - io.swagger swagger-annotations - org.slf4j slf4j-api - - com.fasterxml.jackson.core - jackson-databind + com.fasterxml.jackson.core + jackson-databind - com.google.guava guava - io.netty netty-handler - io.netty netty-resolver-dns - io.netty netty-transport-native-epoll linux-x86_64 - io.netty netty-transport-native-unix-common linux-x86_64 - org.apache.bookkeeper bookkeeper-common-allocator @@ -99,17 +85,14 @@ - org.apache.bookkeeper cpu-affinity - io.airlift aircompressor - org.apache.bookkeeper circe-checksum @@ -125,44 +108,36 @@ - io.netty netty-tcnative-boringssl-static - io.netty netty-codec-haproxy - org.apache.commons commons-lang3 - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - javax.ws.rs javax.ws.rs-api - commons-io commons-io - com.github.spotbugs spotbugs-annotations provided true - org.bouncycastle @@ -170,78 +145,70 @@ ${bouncycastlefips.version} test - org.lz4 lz4-java 1.5.0 test - com.github.luben zstd-jni 1.3.7-3 test - org.xerial.snappy snappy-java 1.1.7.5 test - - com.google.code.gson - gson - - + + com.google.code.gson + gson + com.beust jcommander - org.awaitility awaitility test - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - com.github.splunk.lightproto - lightproto-maven-plugin - ${lightproto-maven-plugin.version} - - - - generate - - - - - - + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + pl.project13.maven git-commit-id-plugin @@ -263,51 +230,50 @@ true - - - - org.codehaus.mojo - templating-maven-plugin - 1.0.0 - - - filtering-java-templates - - filter-sources - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - spotbugs - verify - - check - - - - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - + + + org.codehaus.mojo + templating-maven-plugin + 1.0.0 + + + filtering-java-templates + + filter-sources + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java new file mode 100644 index 0000000000000..024aa153756dc --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/LazyLoadableValue.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import java.util.function.Supplier; + +/*** + * Used to lazy load a value, only calculate it when used. Not thread-safety. + */ +public class LazyLoadableValue { + + private Supplier loader; + + private T value; + + public LazyLoadableValue(Supplier loader) { + this.loader = loader; + } + + public T getValue() { + if (value == null) { + value = loader.get(); + } + return value; + } +} diff --git a/pulsar-common/src/main/resources/findbugsExclude.xml b/pulsar-common/src/main/resources/findbugsExclude.xml index df161c4b621a7..8216948c12408 100644 --- a/pulsar-common/src/main/resources/findbugsExclude.xml +++ b/pulsar-common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-config-validation/pom.xml b/pulsar-config-validation/pom.xml index 1df8526d8a02d..5258bc4c64219 100644 --- a/pulsar-config-validation/pom.xml +++ b/pulsar-config-validation/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 2.10.5-SNAPSHOT - .. - - - pulsar-config-validation - Annotation based config validation for Pulsar - - - - org.slf4j - slf4j-api - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - \ No newline at end of file + + 4.0.0 + + io.streamnative + pulsar + 2.10.5.3 + .. + + pulsar-config-validation + Annotation based config validation for Pulsar + + + org.slf4j + slf4j-api + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + diff --git a/pulsar-functions/api-java/pom.xml b/pulsar-functions/api-java/pom.xml index 47097127dfb80..3b69ddbe1f519 100644 --- a/pulsar-functions/api-java/pom.xml +++ b/pulsar-functions/api-java/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-api Pulsar Functions :: API - - org.slf4j slf4j-api - net.jodah typetools @@ -48,16 +44,13 @@ ${project.version} compile - ${project.groupId} pulsar-client-admin-api ${project.version} compile - - @@ -79,5 +72,4 @@ - diff --git a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/api-java/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/instance/pom.xml b/pulsar-functions/instance/pom.xml index e984ca282c372..da55998594007 100644 --- a/pulsar-functions/instance/pom.xml +++ b/pulsar-functions/instance/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-instance Pulsar Functions :: Instance - - org.apache.logging.log4j @@ -46,61 +42,51 @@ org.apache.logging.log4j log4j-core - ${project.groupId} pulsar-functions-utils ${project.version} - ${project.groupId} pulsar-metadata ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - - + ${project.groupId} pulsar-client-admin-original ${project.version} - - + ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-messagecrypto-bc ${project.parent.version} - org.apache.bookkeeper stream-storage-java-client @@ -115,24 +101,20 @@ - io.grpc grpc-stub - io.grpc grpc-all - io.perfmark perfmark-api runtime - org.apache.bookkeeper bookkeeper-common @@ -147,61 +129,50 @@ - com.yahoo.datasketches sketches-core - com.google.guava guava - com.beust jcommander - net.jodah typetools - io.prometheus simpleclient ${prometheus.version} - io.prometheus simpleclient_hotspot ${prometheus.version} - io.prometheus simpleclient_httpserver ${prometheus.version} - io.prometheus.jmx collector ${prometheus-jmx.version} - org.awaitility awaitility test - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pulsar-functions/java-examples/pom.xml b/pulsar-functions/java-examples/pom.xml index 7283d60ffff9e..84aaadad8e5a4 100644 --- a/pulsar-functions/java-examples/pom.xml +++ b/pulsar-functions/java-examples/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-api-examples Pulsar Functions :: API Examples - org.slf4j slf4j-api - ${project.groupId} pulsar-functions-api @@ -52,7 +49,6 @@ ${project.version} - @@ -74,5 +70,4 @@ - diff --git a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/java-examples/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/localrun-shaded/pom.xml b/pulsar-functions/localrun-shaded/pom.xml index abaf77f45a45e..67a501bf80c63 100644 --- a/pulsar-functions/localrun-shaded/pom.xml +++ b/pulsar-functions/localrun-shaded/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.10.5-SNAPSHOT - .. - - - pulsar-functions-local-runner - Pulsar Functions :: Local Runner Shaded - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.parent.version} - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack - prepare-package - - unpack - - - - - org.asynchttpclient - async-http-client - ${asynchttpclient.version} - jar - true - org/asynchttpclient/config/ahc-default.properties - ${project.build.directory}/classes - - - - - - - - - maven-antrun-plugin - - - shade-ahc-properties - prepare-package - - run - - - - - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - false - - - - - - - org.apache.pulsar:* - org.apache.bookkeeper:* - commons-*:* - org.apache.commons:* - com.fasterxml.jackson.*:* - io.netty:* - com.google.*:* - javax.servlet:* - org.reactivestreams:reactive-streams - org.apache.commons:* - io.swagger:* - org.yaml:snakeyaml - io.perfmark:* - io.prometheus:* - io.prometheus.jmx:* - javax.ws.rs:* - org.tukaani:xz - com.github.zafarkhaja:java-semver - net.java.dev.jna:* - org.apache.zookeeper:* - com.thoughtworks.paranamer:paranamer - jline:* - org.rocksdb:* - org.eclipse.jetty*:* - org.apache.avro:avro - com.beust:* - net.jodah:* - io.airlift:* - com.yahoo.datasketches:* - io.netty.resolver:* - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + false + + + + + + + io.streamnative:* + org.apache.bookkeeper:* + commons-*:* + org.apache.commons:* + com.fasterxml.jackson.*:* + io.netty:* + com.google.*:* + javax.servlet:* + org.reactivestreams:reactive-streams + org.apache.commons:* + io.swagger:* + org.yaml:snakeyaml + io.perfmark:* + io.prometheus:* + io.prometheus.jmx:* + javax.ws.rs:* + org.tukaani:xz + com.github.zafarkhaja:java-semver + net.java.dev.jna:* + org.apache.zookeeper:* + com.thoughtworks.paranamer:paranamer + jline:* + org.rocksdb:* + org.eclipse.jetty*:* + org.apache.avro:avro + com.beust:* + net.jodah:* + io.airlift:* + com.yahoo.datasketches:* + io.netty.resolver:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + - - com.google - org.apache.pulsar.functions.runtime.shaded.com.google - - - org.apache.jute - org.apache.pulsar.functions.runtime.shaded.org.apache.jute - - - javax.servlet - org.apache.pulsar.functions.runtime.shaded.javax.servlet - - - org.junit - org.apache.pulsar.functions.runtime.shaded.org.junit - - - junit - org.apache.pulsar.functions.runtime.shaded.junit - - - net.jodah - org.apache.pulsar.functions.runtime.shaded.net.jodah - - - org.lz4 - org.apache.pulsar.functions.runtime.shaded.org.lz4 - - - org.reactivestreams - org.apache.pulsar.functions.runtime.shaded.org.reactivestreams - - - org.apache.commons - org.apache.pulsar.functions.runtime.shaded.org.apache.commons - - - io.swagger - org.apache.pulsar.functions.runtime.shaded.io.swagger - - - org.yaml - org.apache.pulsar.functions.runtime.shaded.org.yaml - - - org.jctools - org.apache.pulsar.functions.runtime.shaded.org.jctools - - - com.squareup.okhttp - org.apache.pulsar.functions.runtime.shaded.com.squareup.okhttp - - - io.grpc - org.apache.pulsar.functions.runtime.shaded.io.grpc - - - io.perfmark - org.apache.pulsar.functions.runtime.shaded.io.perfmark - - - org.joda - org.apache.pulsar.functions.runtime.shaded.org.joda - - - javax.ws.rs - org.apache.pulsar.functions.runtime.shaded.javax.ws.rs - - - io.kubernetes - org.apache.pulsar.functions.runtime.shaded.io.kubernetes - - - io.opencensus - org.apache.pulsar.functions.runtime.shaded.io.opencensus - - - net.jpountz - org.apache.pulsar.functions.runtime.shaded.net.jpountz - - - commons-configuration - org.apache.pulsar.functions.runtime.shaded.commons-configuration - - - org.tukaani - org.apache.pulsar.functions.runtime.shaded.org.tukaani - - - com.github - org.apache.pulsar.functions.runtime.shaded.com.github - - - commons-io - org.apache.pulsar.functions.runtime.shaded.commons-io - - - org.apache.distributedlog - org.apache.pulsar.functions.runtime.shaded.org.apache.distributedlog - - - - com.fasterxml - org.apache.pulsar.functions.runtime.shaded.com.fasterxml - - - org.inferred - org.apache.pulsar.functions.runtime.shaded.org.inferred - - - org.apache.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - - - org.bookkeeper - org.apache.pulsar.functions.runtime.shaded.org.bookkeeper - - - dlshade - org.apache.pulsar.functions.runtime.shaded.dlshade - - - - org.codehaus.jackson - org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson - - - net.java.dev.jna - org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna - - - org.apache.curator - org.apache.pulsar.functions.runtime.shaded.org.apache.curator - - - javax.validation - org.apache.pulsar.functions.runtime.shaded.javax.validation - - - javax.activation - org.apache.pulsar.functions.runtime.shaded.javax.activation - - - io.prometheus - org.apache.pulsar.functions.runtime.shaded.io.prometheus - - - io.prometheus.jmx - org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx - - - org.apache.zookeeper - org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper - - - io.jsonwebtoken - org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken - - - commons-codec - org.apache.pulsar.functions.runtime.shaded.commons-codec - - - com.thoughtworks.paranamer - org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer - - - org.codehaus.mojo - org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo - - - com.github.luben - org.apache.pulsar.functions.runtime.shaded.com.github.luben - - - jline - org.apache.pulsar.functions.runtime.shaded.jline - - - org.xerial.snappy - org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy - - - javax.annotation - org.apache.pulsar.functions.runtime.shaded.javax.annotation - - - org.checkerframework - org.apache.pulsar.functions.runtime.shaded.org.checkerframework - - - org.apache.yetus - org.apache.pulsar.functions.runtime.shaded.org.apache.yetus - - - commons-cli - org.apache.pulsar.functions.runtime.shaded.commons-cli - - - commons-lang - org.apache.pulsar.functions.runtime.shaded.commons-lang - - - com.squareup.okio - org.apache.pulsar.functions.runtime.shaded.com.squareup.okio - - - org.rocksdb - org.apache.pulsar.functions.runtime.shaded.org.rocksdb - - - org.objenesis - org.apache.pulsar.functions.runtime.shaded.org.objenesis - - - org.eclipse.jetty - org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty - - - org.apache.avro - org.apache.pulsar.functions.runtime.shaded.org.apache.avro - - - avro.shaded - org.apache.pulsar.functions.runtime.shaded.avo.shaded - - - com.yahoo.datasketches - org.apache.pulsar.shaded.com.yahoo.datasketches - - - com.yahoo.sketches - org.apache.pulsar.shaded.com.yahoo.sketches - - - com.beust - org.apache.pulsar.functions.runtime.shaded.com.beust - - - - io.netty - org.apache.pulsar.functions.runtime.shaded.io.netty - - - org.hamcrest - org.apache.pulsar.functions.runtime.shaded.org.hamcrest - - - aj.org - org.apache.pulsar.functions.runtime.shaded.aj.org - - - com.scurrilous - org.apache.pulsar.functions.runtime.shaded.com.scurrilous - - - okio - org.apache.pulsar.functions.runtime.shaded.okio - - + + com.fasterxml + org.apache.pulsar.functions.runtime.shaded.com.fasterxml + + + org.inferred + org.apache.pulsar.functions.runtime.shaded.org.inferred + + + org.apache.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + + + org.bookkeeper + org.apache.pulsar.functions.runtime.shaded.org.bookkeeper + + + dlshade + org.apache.pulsar.functions.runtime.shaded.dlshade + + + + org.codehaus.jackson + org.apache.pulsar.functions.runtime.shaded.org.codehaus.jackson + + + net.java.dev.jna + org.apache.pulsar.functions.runtime.shaded.net.java.dev.jna + + + org.apache.curator + org.apache.pulsar.functions.runtime.shaded.org.apache.curator + + + javax.validation + org.apache.pulsar.functions.runtime.shaded.javax.validation + + + javax.activation + org.apache.pulsar.functions.runtime.shaded.javax.activation + + + io.prometheus + org.apache.pulsar.functions.runtime.shaded.io.prometheus + + + io.prometheus.jmx + org.apache.pulsar.functions.runtime.shaded.io.prometheus.jmx + + + org.apache.zookeeper + org.apache.pulsar.functions.runtime.shaded.org.apache.zookeeper + + + io.jsonwebtoken + org.apache.pulsar.functions.runtime.shaded.io.jsonwebtoken + + + commons-codec + org.apache.pulsar.functions.runtime.shaded.commons-codec + + + com.thoughtworks.paranamer + org.apache.pulsar.functions.runtime.shaded.com.thoughtworks.paranamer + + + org.codehaus.mojo + org.apache.pulsar.functions.runtime.shaded.org.codehaus.mojo + + + com.github.luben + org.apache.pulsar.functions.runtime.shaded.com.github.luben + + + jline + org.apache.pulsar.functions.runtime.shaded.jline + + + org.xerial.snappy + org.apache.pulsar.functions.runtime.shaded.org.xerial.snappy + + + javax.annotation + org.apache.pulsar.functions.runtime.shaded.javax.annotation + + + org.checkerframework + org.apache.pulsar.functions.runtime.shaded.org.checkerframework + + + org.apache.yetus + org.apache.pulsar.functions.runtime.shaded.org.apache.yetus + + + commons-cli + org.apache.pulsar.functions.runtime.shaded.commons-cli + + + commons-lang + org.apache.pulsar.functions.runtime.shaded.commons-lang + + + com.squareup.okio + org.apache.pulsar.functions.runtime.shaded.com.squareup.okio + + + org.rocksdb + org.apache.pulsar.functions.runtime.shaded.org.rocksdb + + + org.objenesis + org.apache.pulsar.functions.runtime.shaded.org.objenesis + + + org.eclipse.jetty + org.apache.pulsar.functions.runtime.shaded.org.eclipse.jetty + + + org.apache.avro + org.apache.pulsar.functions.runtime.shaded.org.apache.avro + + + avro.shaded + org.apache.pulsar.functions.runtime.shaded.avo.shaded + + + com.yahoo.datasketches + org.apache.pulsar.shaded.com.yahoo.datasketches + + + com.yahoo.sketches + org.apache.pulsar.shaded.com.yahoo.sketches + + + com.beust + org.apache.pulsar.functions.runtime.shaded.com.beust + + + + io.netty + org.apache.pulsar.functions.runtime.shaded.io.netty + + + org.hamcrest + org.apache.pulsar.functions.runtime.shaded.org.hamcrest + + + aj.org + org.apache.pulsar.functions.runtime.shaded.aj.org + + + com.scurrilous + org.apache.pulsar.functions.runtime.shaded.com.scurrilous + + + okio + org.apache.pulsar.functions.runtime.shaded.okio + + - - org.asynchttpclient - org.apache.pulsar.functions.runtime.shaded.org.asynchttpclient - - - io.airlift - org.apache.pulsar.functions.runtime.shaded.io.airlift - - - - - - - - - + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-functions/localrun/pom.xml b/pulsar-functions/localrun/pom.xml index 2744b5aa51c75..5552e412e7313 100644 --- a/pulsar-functions/localrun/pom.xml +++ b/pulsar-functions/localrun/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.10.5-SNAPSHOT - .. - - - pulsar-functions-local-runner-original - Pulsar Functions :: Local Runner original - - - - ${project.groupId} - pulsar-functions-runtime - ${project.parent.version} - - - - - org.apache.logging.log4j - log4j-slf4j-impl - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - - io.grpc - grpc-all - - - - io.perfmark - perfmark-api - runtime - - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - check - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 2.10.5.3 + .. + + pulsar-functions-local-runner-original + Pulsar Functions :: Local Runner original + + + ${project.groupId} + pulsar-functions-runtime + ${project.parent.version} + + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + io.grpc + grpc-all + + + io.perfmark + perfmark-api + runtime + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + check + verify + + check + + + + + + diff --git a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml index 719b711d6560a..02157e2e9ea6f 100644 --- a/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/localrun/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - \ No newline at end of file + + + + + diff --git a/pulsar-functions/localrun/src/main/resources/log4j2.xml b/pulsar-functions/localrun/src/main/resources/log4j2.xml index a7714c3f48835..aa28dda160b02 100644 --- a/pulsar-functions/localrun/src/main/resources/log4j2.xml +++ b/pulsar-functions/localrun/src/main/resources/log4j2.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + diff --git a/pulsar-functions/pom.xml b/pulsar-functions/pom.xml index 02bb1210b092e..3f4119dcde70b 100644 --- a/pulsar-functions/pom.xml +++ b/pulsar-functions/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions Pulsar Functions :: Parent - main @@ -41,23 +39,22 @@ !true - - proto - api-java - java-examples - utils - instance - runtime - runtime-all - worker - secrets - localrun - localrun-shaded - + + proto + api-java + java-examples + utils + instance + runtime + runtime-all + worker + secrets + localrun + localrun-shaded + - - - core-modules + + core-modules proto api-java @@ -70,8 +67,6 @@ secrets localrun - - + - diff --git a/pulsar-functions/proto/pom.xml b/pulsar-functions/proto/pom.xml index beb1ab2d5be93..6a05246337f4d 100644 --- a/pulsar-functions/proto/pom.xml +++ b/pulsar-functions/proto/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-functions - 2.10.5-SNAPSHOT - - - pulsar-functions-proto - Pulsar Functions :: Proto - - - - - com.google.protobuf - protobuf-java - - - - com.google.protobuf - protobuf-java-util - - - - javax.annotation - javax.annotation-api - - - - io.grpc - grpc-all - - - io.netty - * - - - - - - io.perfmark - perfmark-api - runtime - - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - ${protobuf-maven-plugin.version} - - com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} - true - grpc-java - io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} - - - - generate-sources - - compile - compile-custom - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-functions + 2.10.5.3 + + pulsar-functions-proto + Pulsar Functions :: Proto + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + javax.annotation + javax.annotation-api + + + io.grpc + grpc-all + + + io.netty + * + + + + + io.perfmark + perfmark-api + runtime + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protoc3.version}:exe:${os.detected.classifier} + true + grpc-java + io.grpc:protoc-gen-grpc-java:${protoc-gen-grpc-java.version}:exe:${os.detected.classifier} + + + + generate-sources + + compile + compile-custom + + + + + + diff --git a/pulsar-functions/runtime-all/pom.xml b/pulsar-functions/runtime-all/pom.xml index 06bede1abfb5e..519785ecac698 100644 --- a/pulsar-functions/runtime-all/pom.xml +++ b/pulsar-functions/runtime-all/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 .. - - pulsar-functions-runtime-all Pulsar Functions :: Runtime All - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-client-api ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.protobuf @@ -94,30 +84,24 @@ gson ${gson.version} - - org.slf4j slf4j-api - org.apache.logging.log4j log4j-slf4j-impl - org.apache.logging.log4j log4j-api - org.apache.logging.log4j log4j-core - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml index 190d9be92940b..84e6ce44f51e4 100644 --- a/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/java_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-instance - 30 - - - pulsar.log.appender - RollingFile - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - RollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}*log.gz - - - 30d - - - - - - BkRollingFile - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk - ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz - true - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - 1 - true - - - 1 GB - - - 0 0 0 * * ? - - - - - ${sys:pulsar.function.log.dir} - 2 - - */${sys:pulsar.function.log.file}.bk*log.gz - - - 30d - - - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - BkRollingFile - - - - info - - ${sys:pulsar.log.appender} - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-instance + 30 + + + pulsar.log.appender + RollingFile + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + RollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.log + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}*log.gz + + + 30d + + + + + + BkRollingFile + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk + ${sys:pulsar.function.log.dir}/${sys:pulsar.function.log.file}.bk-%d{MM-dd-yyyy}-%i.log.gz + true + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + 1 + true + + + 1 GB + + + 0 0 0 * * ? + + + + + ${sys:pulsar.function.log.dir} + 2 + + */${sys:pulsar.function.log.file}.bk*log.gz + + + 30d + + + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + BkRollingFile + + + + info + + ${sys:pulsar.log.appender} + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml index f86d03e41793f..fe478be5b8667 100644 --- a/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml +++ b/pulsar-functions/runtime-all/src/main/resources/kubernetes_instance_log4j2.xml @@ -1,3 +1,4 @@ + - pulsar-functions-kubernetes-instance - 30 - - - pulsar.log.level - info - - - bk.log.level - info - - - - - Console - SYSTEM_OUT - - %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n - - - - - - org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper - ${sys:bk.log.level} - false - - Console - - - - info - - Console - ${sys:pulsar.log.level} - - - - \ No newline at end of file + pulsar-functions-kubernetes-instance + 30 + + + pulsar.log.level + info + + + bk.log.level + info + + + + + Console + SYSTEM_OUT + + %d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n + + + + + + org.apache.pulsar.functions.runtime.shaded.org.apache.bookkeeper + ${sys:bk.log.level} + false + + Console + + + + info + + Console + ${sys:pulsar.log.level} + + + + diff --git a/pulsar-functions/runtime/pom.xml b/pulsar-functions/runtime/pom.xml index afc7b73c07cf7..dc0f3ab3d5ff0 100644 --- a/pulsar-functions/runtime/pom.xml +++ b/pulsar-functions/runtime/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-runtime Pulsar Functions :: Runtime - - ${project.groupId} pulsar-functions-instance ${project.version} - ${project.groupId} pulsar-functions-secrets ${project.version} - com.beust jcommander - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-databind - io.kubernetes client-java @@ -70,9 +61,7 @@ pulsar-broker-common ${project.version} - - @@ -118,5 +107,4 @@ - diff --git a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml index b4a4e5926de3f..976abe7c54bb2 100644 --- a/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/runtime/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - \ No newline at end of file + + + + + + + + + + diff --git a/pulsar-functions/secrets/pom.xml b/pulsar-functions/secrets/pom.xml index 7712da5ffbc71..f3a54c847b9c9 100644 --- a/pulsar-functions/secrets/pom.xml +++ b/pulsar-functions/secrets/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-secrets Pulsar Functions :: Secrets - io.kubernetes @@ -50,7 +48,6 @@ - ${project.groupId} pulsar-functions-proto @@ -78,5 +75,4 @@ - diff --git a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/secrets/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-functions/utils/pom.xml b/pulsar-functions/utils/pom.xml index 1ccdd44f56161..aa6cf7036e6db 100644 --- a/pulsar-functions/utils/pom.xml +++ b/pulsar-functions/utils/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-functions-utils Pulsar Functions :: Utils - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-proto ${project.version} - ${project.groupId} pulsar-functions-api ${project.version} - ${project.groupId} pulsar-config-validation ${project.version} - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.core jackson-databind - com.google.code.gson gson - net.jodah typetools - - ${project.groupId} - pulsar-client-original - ${project.version} + ${project.groupId} + pulsar-client-original + ${project.version} - ${project.groupId} pulsar-package-core ${project.version} - - diff --git a/pulsar-functions/worker/pom.xml b/pulsar-functions/worker/pom.xml index 31debbdf14e02..b2f3beff76803 100644 --- a/pulsar-functions/worker/pom.xml +++ b/pulsar-functions/worker/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-functions - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-functions-worker Pulsar Functions :: Worker - - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-functions-runtime ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-client-admin-original ${project.version} - org.glassfish.jersey.media jersey-media-json-jackson - jakarta.activation jakarta.activation-api - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-multipart - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.apache.distributedlog distributedlog-core @@ -113,17 +96,14 @@ - org.apache.bookkeeper bookkeeper-server - commons-io commons-io - io.swagger swagger-core @@ -138,14 +118,11 @@ - io.prometheus simpleclient_jetty - - ${project.groupId} pulsar-io-cassandra @@ -153,7 +130,6 @@ pom test - ${project.groupId} pulsar-io-twitter @@ -161,7 +137,6 @@ pom test - ${project.groupId} pulsar-functions-api-examples @@ -169,9 +144,7 @@ pom test - - @@ -217,7 +190,6 @@ - org.apache.maven.plugins maven-surefire-plugin @@ -232,7 +204,6 @@ - org.apache.maven.plugins @@ -264,17 +235,16 @@ ${basedir}/src/main/resources/findbugsExclude.xml - - - - - - - - - + + + + + + + + + - diff --git a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-functions/worker/src/main/resources/findbugsExclude.xml +++ b/pulsar-functions/worker/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml index df6e964c02483..5406c1868bad6 100644 --- a/pulsar-io/aerospike/pom.xml +++ b/pulsar-io/aerospike/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-aerospike Pulsar IO :: Aerospike - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.aerospike aerospike-client-bc ${aerospike-client.version} - - @@ -81,6 +73,4 @@ - - diff --git a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aerospike/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/aws/pom.xml b/pulsar-io/aws/pom.xml index 634ed92122bab..ad3aa3adcbadf 100644 --- a/pulsar-io/aws/pom.xml +++ b/pulsar-io/aws/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-aws Pulsar IO :: IO AWS - ${project.groupId} pulsar-io-core ${project.version} - com.google.code.gson gson - com.amazonaws aws-java-sdk-sts - software.amazon.awssdk sts 2.10.56 - - + - diff --git a/pulsar-io/aws/src/main/resources/findbugsExclude.xml b/pulsar-io/aws/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/aws/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/aws/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/batch-data-generator/pom.xml b/pulsar-io/batch-data-generator/pom.xml index b191f2be714d1..99fd1389abe9e 100644 --- a/pulsar-io/batch-data-generator/pom.xml +++ b/pulsar-io/batch-data-generator/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - - pulsar-io-batch-data-generator - Pulsar IO :: Batch Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-io-batch-discovery-triggerers - ${project.version} - - - - org.springframework - spring-context - ${spring-context.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-functions-local-runner-original - ${project.version} - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 2.10.5.3 + + pulsar-io-batch-data-generator + Pulsar IO :: Batch Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-io-batch-discovery-triggerers + ${project.version} + + + org.springframework + spring-context + ${spring-context.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-functions-local-runner-original + ${project.version} + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-data-generator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-batch-discovery-triggerers Pulsar IO :: Batch Discovery Triggerers - ${project.groupId} pulsar-io-core ${project.version} - com.cronutils cron-utils ${cron-utils.version} - org.springframework spring-context ${spring-context.version} - - @@ -73,5 +66,4 @@ - diff --git a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/batch-discovery-triggerers/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - 4.0.0 - - pulsar-io-canal - Pulsar IO :: Canal - - - 5.3.19 - 1.1.5 - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.alibaba - fastjson - 1.2.83 - - - - org.springframework - spring-core - ${spring.version} - - - org.springframework - spring-aop - ${spring.version} - - - org.springframework - spring-context - ${spring.version} - - - org.springframework - spring-jdbc - ${spring.version} - - - org.springframework - spring-orm - ${spring.version} - - - - com.alibaba.otter - canal.protocol - ${canal.version} - - - com.alibaba.otter - canal.client - ${canal.version} - - - ch.qos.logback - * - - - org.springframework - * - - - - - - org.apache.logging.log4j - log4j-core - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - \ No newline at end of file + + + io.streamnative + pulsar-io + 2.10.5.3 + + 4.0.0 + pulsar-io-canal + Pulsar IO :: Canal + + 5.3.19 + 1.1.5 + + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + com.alibaba + fastjson + 1.2.83 + + + org.springframework + spring-core + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-context + ${spring.version} + + + org.springframework + spring-jdbc + ${spring.version} + + + org.springframework + spring-orm + ${spring.version} + + + com.alibaba.otter + canal.protocol + ${canal.version} + + + com.alibaba.otter + canal.client + ${canal.version} + + + ch.qos.logback + * + + + org.springframework + * + + + + + org.apache.logging.log4j + log4j-core + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/cassandra/pom.xml b/pulsar-io/cassandra/pom.xml index 901f752771cce..556bc8d9f18e5 100644 --- a/pulsar-io/cassandra/pom.xml +++ b/pulsar-io/cassandra/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-cassandra Pulsar IO :: Cassandra - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.datastax.cassandra cassandra-driver-core ${cassandra-driver-core.version} - - diff --git a/pulsar-io/common/pom.xml b/pulsar-io/common/pom.xml index 1517ad01d3df8..e8845067c04d6 100644 --- a/pulsar-io/common/pom.xml +++ b/pulsar-io/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - - pulsar-io-common - Pulsar IO :: IO Common - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-common - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 2.10.5.3 + + pulsar-io-common + Pulsar IO :: IO Common + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-common + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/common/src/main/resources/findbugsExclude.xml b/pulsar-io/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-core Pulsar IO :: IO - ${project.groupId} @@ -37,7 +35,6 @@ ${project.version} - @@ -59,5 +56,4 @@ - diff --git a/pulsar-io/core/src/main/resources/findbugsExclude.xml b/pulsar-io/core/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - - pulsar-io-data-generator - Pulsar IO :: Data Generator - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-config-validation - ${project.version} - - - - io.codearte.jfairy - jfairy - 0.5.9 - - - - org.apache.avro - avro - ${avro.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - + + 4.0.0 + + io.streamnative + pulsar-io + 2.10.5.3 + + pulsar-io-data-generator + Pulsar IO :: Data Generator + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-config-validation + ${project.version} + + + io.codearte.jfairy + jfairy + 0.5.9 + + + org.apache.avro + avro + ${avro.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/core/pom.xml b/pulsar-io/debezium/core/pom.xml index c68c37a607b3c..13af929d5cb84 100644 --- a/pulsar-io/debezium/core/pom.xml +++ b/pulsar-io/debezium/core/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium-core Pulsar IO :: Debezium :: Core - - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - io.debezium debezium-core ${debezium.version} - ${project.groupId} pulsar-io-kafka-connect-adaptor ${project.version} - org.apache.kafka kafka_${scala.binary.version} ${kafka-client.version} - org.apache.kafka connect-runtime @@ -73,21 +65,18 @@ - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -95,14 +84,11 @@ test test-jar - io.debezium debezium-connector-mysql ${debezium.version} test - - diff --git a/pulsar-io/debezium/mongodb/pom.xml b/pulsar-io/debezium/mongodb/pom.xml index 76170a06ba380..277ed6732cfeb 100644 --- a/pulsar-io/debezium/mongodb/pom.xml +++ b/pulsar-io/debezium/mongodb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io-debezium - 2.10.5-SNAPSHOT - - - pulsar-io-debezium-mongodb - Pulsar IO :: Debezium :: mongodb - - - - - ${project.groupId} - pulsar-io-debezium-core - ${project.version} - - - - - org.mongodb - mongodb-driver-sync - 4.2.2 - - - - io.debezium - debezium-connector-mongodb - ${debezium.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - + + 4.0.0 + + io.streamnative + pulsar-io-debezium + 2.10.5.3 + + pulsar-io-debezium-mongodb + Pulsar IO :: Debezium :: mongodb + + + ${project.groupId} + pulsar-io-debezium-core + ${project.version} + + + + org.mongodb + mongodb-driver-sync + 4.2.2 + + + io.debezium + debezium-connector-mongodb + ${debezium.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + diff --git a/pulsar-io/debezium/mssql/pom.xml b/pulsar-io/debezium/mssql/pom.xml index 3a722020cabf0..8ad1b65449615 100644 --- a/pulsar-io/debezium/mssql/pom.xml +++ b/pulsar-io/debezium/mssql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium-mssql Pulsar IO :: Debezium :: Microsoft SQL - - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-sqlserver ${debezium.version} - - @@ -54,5 +48,4 @@ - diff --git a/pulsar-io/debezium/mysql/pom.xml b/pulsar-io/debezium/mysql/pom.xml index a5bd266612740..21909e1d3ebb1 100644 --- a/pulsar-io/debezium/mysql/pom.xml +++ b/pulsar-io/debezium/mysql/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium-mysql Pulsar IO :: Debezium :: mysql - - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-mysql ${debezium.version} - - @@ -55,8 +49,6 @@ - - @@ -65,5 +57,4 @@ - diff --git a/pulsar-io/debezium/oracle/pom.xml b/pulsar-io/debezium/oracle/pom.xml index 75cdf971c414e..71a75687eddd0 100644 --- a/pulsar-io/debezium/oracle/pom.xml +++ b/pulsar-io/debezium/oracle/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium-oracle Pulsar IO :: Debezium :: oracle - - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-oracle ${debezium.version} - - @@ -54,5 +48,4 @@ - diff --git a/pulsar-io/debezium/pom.xml b/pulsar-io/debezium/pom.xml index 53890b23fb617..6c0b2e5be1a0d 100644 --- a/pulsar-io/debezium/pom.xml +++ b/pulsar-io/debezium/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium Pulsar IO :: Debezium - @@ -70,7 +68,6 @@ - core mysql @@ -79,5 +76,4 @@ oracle mssql - diff --git a/pulsar-io/debezium/postgres/pom.xml b/pulsar-io/debezium/postgres/pom.xml index bb91815add537..b49c9fba982a2 100644 --- a/pulsar-io/debezium/postgres/pom.xml +++ b/pulsar-io/debezium/postgres/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io-debezium - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-debezium-postgres Pulsar IO :: Debezium :: postgres - - ${project.groupId} pulsar-io-debezium-core ${project.version} - io.debezium debezium-connector-postgres ${debezium.version} - org.postgresql postgresql ${debezium.postgresql.version} runtime - - @@ -61,5 +54,4 @@ - diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml index 64ff9e833d0e5..6273e723c9350 100644 --- a/pulsar-io/docs/pom.xml +++ b/pulsar-io/docs/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-docs Pulsar IO :: Docs - - ${project.groupId} pulsar-io-core @@ -45,7 +42,6 @@ com.beust jcommander - ${project.groupId} @@ -193,7 +189,6 @@ ${project.version} - @@ -242,5 +237,4 @@ - diff --git a/pulsar-io/docs/src/main/resources/findbugsExclude.xml b/pulsar-io/docs/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-io/docs/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/docs/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-dynamodb Pulsar IO :: DynamoDB - - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-instance ${project.version} provided - ${project.groupId} pulsar-io-aws ${project.version} - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - com.amazonaws @@ -90,9 +78,7 @@ 1.5.1 - - @@ -101,5 +87,4 @@ - diff --git a/pulsar-io/elastic-search/pom.xml b/pulsar-io/elastic-search/pom.xml index 8dc03450361ea..c25dbcbc7c27f 100644 --- a/pulsar-io/elastic-search/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - - pulsar-io-flume - Pulsar IO :: Flume - - - 1.8.2 - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - org.apache.flume - flume-ng-node - 1.9.0 - pom - - - org.apache.avro - avro-ipc - - - avro - org.apache.avro - - - - - org.apache.avro - avro - ${avro.version} - - - commons-lang - commons-lang - 2.6 - - - org.apache.avro - avro-ipc - ${avro.version} - - - org.mortbay.jetty - servlet-api - - - io.netty - netty - - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-test - ${curator.version} - test - - - - io.dropwizard.metrics - metrics-core - test - - - - org.xerial.snappy - snappy-java - test - - - - com.google.guava - guava - ${guava.version} - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + com.google.guava + guava + ${guava.version} + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/pom.xml b/pulsar-io/hbase/pom.xml index a333ddebd2006..b6eade11fc55d 100644 --- a/pulsar-io/hbase/pom.xml +++ b/pulsar-io/hbase/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 2.10.5-SNAPSHOT - - pulsar-io-hbase - Pulsar IO :: Hbase - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - com.google.guava - guava - - - - org.apache.hbase - hbase-client - ${hbase.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hbase - hbase-common - ${hbase.version} - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml index c2b520a765643..460cbd497c15f 100644 --- a/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml +++ b/pulsar-io/hbase/src/test/resources/hbase/hbase-site.xml @@ -1,3 +1,4 @@ + - - hbase.cluster.distributed - true - - - hbase.rootdir - hdfs://localhost:8020/hbase - - - hbase.zookeeper.quorum - localhost - - - hbase.zookeeper.property.clientPort - 2181 - - - zookeeper.znode.parent - /hbase - + + hbase.cluster.distributed + true + + + hbase.rootdir + hdfs://localhost:8020/hbase + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + + zookeeper.znode.parent + /hbase + diff --git a/pulsar-io/hdfs2/pom.xml b/pulsar-io/hdfs2/pom.xml index 3843bbd7c250b..7c45849569072 100644 --- a/pulsar-io/hdfs2/pom.xml +++ b/pulsar-io/hdfs2/pom.xml @@ -1,3 +1,4 @@ + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - - \ No newline at end of file + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + + diff --git a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml index ddde8120ba518..47c3d73af5f06 100644 --- a/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/hdfs2/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml index 31d1e98c47558..b5cf3c90fbe40 100644 --- a/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml +++ b/pulsar-io/hdfs2/src/test/resources/hadoop/core-site.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs2/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/hdfs3/pom.xml b/pulsar-io/hdfs3/pom.xml index 5d14934b49305..eac43e4bf1470 100644 --- a/pulsar-io/hdfs3/pom.xml +++ b/pulsar-io/hdfs3/pom.xml @@ -1,3 +1,4 @@ + - \ No newline at end of file + diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml index 31d1e98c47558..b5cf3c90fbe40 100644 --- a/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml +++ b/pulsar-io/hdfs3/src/test/resources/hadoop/core-site.xml @@ -1,3 +1,4 @@ + - - fs.defaultFS - hdfs://0.0.0.0:8020 - - - io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec, + + fs.defaultFS + hdfs://0.0.0.0:8020 + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec, org.apache.hadoop.io.compress.DefaultCodec, org.apache.hadoop.io.compress.SnappyCodec - + diff --git a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml index bb722f1f63470..1b35ebd585012 100644 --- a/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml +++ b/pulsar-io/hdfs3/src/test/resources/hadoop/hdfs-site.xml @@ -1,3 +1,4 @@ + - - dfs.replication - 1 - - - dfs.client.use.datanode.hostname - true - - - dfs.support.append - true - + + dfs.replication + 1 + + + dfs.client.use.datanode.hostname + true + + + dfs.support.append + true + diff --git a/pulsar-io/influxdb/pom.xml b/pulsar-io/influxdb/pom.xml index 3e7420a364ce0..ff60bbe313d79 100644 --- a/pulsar-io/influxdb/pom.xml +++ b/pulsar-io/influxdb/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 2.10.5-SNAPSHOT - - - pulsar-io-influxdb - Pulsar IO :: InfluxDB - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - com.influxdb - influxdb-client-java - 4.0.0 - - - - org.influxdb - influxdb-java - 2.22 - - - com.squareup.okhttp3 - * - - - - - - org.apache.commons - commons-lang3 - 3.4 - - - commons-collections - commons-collections - 3.2.2 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 2.10.5.3 + + pulsar-io-influxdb + Pulsar IO :: InfluxDB + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + com.influxdb + influxdb-client-java + 4.0.0 + + + org.influxdb + influxdb-java + 2.22 + + + com.squareup.okhttp3 + * + + + + + org.apache.commons + commons-lang3 + 3.4 + + + commons-collections + commons-collections + 3.2.2 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/jdbc/clickhouse/pom.xml b/pulsar-io/jdbc/clickhouse/pom.xml index f3b0abb1b4f6b..66897e888cba9 100644 --- a/pulsar-io/jdbc/clickhouse/pom.xml +++ b/pulsar-io/jdbc/clickhouse/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 4.0.0 - pulsar-io-jdbc-clickhouse Pulsar IO :: Jdbc :: ClickHouse - ${project.groupId} @@ -44,7 +41,6 @@ runtime - diff --git a/pulsar-io/jdbc/core/pom.xml b/pulsar-io/jdbc/core/pom.xml index 944b2962ab68a..248407aa47bb6 100644 --- a/pulsar-io/jdbc/core/pom.xml +++ b/pulsar-io/jdbc/core/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 4.0.0 - pulsar-io-jdbc-core Pulsar IO :: Jdbc :: Core - ${project.groupId} pulsar-io-core ${project.version} - org.apache.avro avro ${avro.version} - com.fasterxml.jackson.core jackson-databind - com.google.guava guava - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - \ No newline at end of file + diff --git a/pulsar-io/jdbc/mariadb/pom.xml b/pulsar-io/jdbc/mariadb/pom.xml index c7127fea55183..295b05cdb1d81 100644 --- a/pulsar-io/jdbc/mariadb/pom.xml +++ b/pulsar-io/jdbc/mariadb/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 4.0.0 - pulsar-io-jdbc-mariadb Pulsar IO :: Jdbc :: Mariadb - ${project.groupId} @@ -52,4 +49,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml index f9f774f327f48..d97bbc86a7580 100644 --- a/pulsar-io/jdbc/pom.xml +++ b/pulsar-io/jdbc/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom @@ -30,12 +30,10 @@ sqlite - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-jdbc Pulsar IO :: Jdbc - diff --git a/pulsar-io/jdbc/postgres/pom.xml b/pulsar-io/jdbc/postgres/pom.xml index 0fbcdc2c25b06..62e25338f6fe1 100644 --- a/pulsar-io/jdbc/postgres/pom.xml +++ b/pulsar-io/jdbc/postgres/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 4.0.0 - pulsar-io-jdbc-postgres Pulsar IO :: Jdbc :: Postgres - ${project.groupId} @@ -44,7 +41,6 @@ runtime - diff --git a/pulsar-io/jdbc/sqlite/pom.xml b/pulsar-io/jdbc/sqlite/pom.xml index 257e5c73eb304..b62ee375bc7f6 100644 --- a/pulsar-io/jdbc/sqlite/pom.xml +++ b/pulsar-io/jdbc/sqlite/pom.xml @@ -1,3 +1,4 @@ + - + pulsar-io-jdbc - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 4.0.0 pulsar-io-jdbc-sqlite Pulsar IO :: Jdbc :: Sqlite - ${project.groupId} @@ -55,7 +53,6 @@ test - @@ -64,4 +61,4 @@ - \ No newline at end of file + diff --git a/pulsar-io/kafka-connect-adaptor-nar/pom.xml b/pulsar-io/kafka-connect-adaptor-nar/pom.xml index a14a5d899b104..490cefdf358a3 100644 --- a/pulsar-io/kafka-connect-adaptor-nar/pom.xml +++ b/pulsar-io/kafka-connect-adaptor-nar/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-kafka-connect-adaptor-nar Pulsar IO :: Kafka Connect Adaptor NAR - ${project.groupId} @@ -37,8 +35,6 @@ ${project.version} - - @@ -48,7 +44,30 @@ pulsar-io-kafka-connect-adaptor-${project.version} + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + - diff --git a/pulsar-io/kafka-connect-adaptor/pom.xml b/pulsar-io/kafka-connect-adaptor/pom.xml index bc7b9a18c7a5d..57a59106f6106 100644 --- a/pulsar-io/kafka-connect-adaptor/pom.xml +++ b/pulsar-io/kafka-connect-adaptor/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-kafka-connect-adaptor Pulsar IO :: Kafka Connect Adaptor - - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-io-common ${project.version} - org.apache.kafka kafka_${scala.binary.version} ${kafka-client.version} - org.apache.kafka connect-runtime @@ -61,25 +55,21 @@ - org.apache.kafka connect-json ${kafka-client.version} - org.apache.kafka connect-api ${kafka-client.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} kafka-connect-avro-converter-shaded @@ -95,28 +85,24 @@ - ${project.groupId} pulsar-broker ${project.version} test - org.apache.kafka connect-file ${kafka-client.version} test - ${project.groupId} testmocks ${project.version} test - ${project.groupId} pulsar-broker @@ -124,12 +110,15 @@ test test-jar - com.google.protobuf protobuf-java provided - + + com.google.protobuf + protobuf-java + 3.5.1 + diff --git a/pulsar-io/kafka/pom.xml b/pulsar-io/kafka/pom.xml index 863e780d7e44c..af4c805b79244 100644 --- a/pulsar-io/kafka/pom.xml +++ b/pulsar-io/kafka/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-kafka Pulsar IO :: Kafka - @@ -44,63 +42,51 @@ - - ${project.groupId} pulsar-io-core ${project.version} provided - ${project.groupId} pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - org.apache.kafka kafka-clients ${kafka-client.version} - io.confluent kafka-schema-registry-client ${kafka.confluent.schemaregistryclient.version} - io.confluent kafka-avro-serializer ${kafka.confluent.avroserializer.version} - org.awaitility awaitility test - - diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java index 3933375512baf..2c4c3dd3de5a7 100644 --- a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaAbstractSource.java @@ -26,7 +26,6 @@ import java.util.Optional; import java.util.Properties; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -43,7 +42,6 @@ import org.apache.pulsar.common.schema.KeyValueEncodingType; import org.apache.pulsar.functions.api.KVRecord; import org.apache.pulsar.functions.api.Record; -import org.apache.pulsar.io.core.PushSource; import org.apache.pulsar.io.core.SourceContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +49,7 @@ /** * Simple Kafka Source to transfer messages from a Kafka topic. */ -public abstract class KafkaAbstractSource extends PushSource { +public abstract class KafkaAbstractSource extends KafkaPushSource { private static final Logger LOG = LoggerFactory.getLogger(KafkaAbstractSource.class); @@ -126,7 +124,6 @@ public void open(Map config, SourceContext sourceContext) throws throw new IllegalArgumentException("Unable to instantiate Kafka consumer", ex); } this.start(); - running = true; } protected Properties beforeCreateConsumer(Properties props) { @@ -151,47 +148,36 @@ public void close() throws InterruptedException { @SuppressWarnings("unchecked") public void start() { + LOG.info("Starting subscribe kafka source on {}", kafkaSourceConfig.getTopic()); + consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic())); runnerThread = new Thread(() -> { - LOG.info("Starting kafka source on {}", kafkaSourceConfig.getTopic()); - consumer.subscribe(Collections.singletonList(kafkaSourceConfig.getTopic())); LOG.info("Kafka source started."); while (running) { - ConsumerRecords consumerRecords = consumer.poll(Duration.ofSeconds(1L)); - CompletableFuture[] futures = new CompletableFuture[consumerRecords.count()]; - int index = 0; - for (ConsumerRecord consumerRecord : consumerRecords) { - KafkaRecord record = buildRecord(consumerRecord); - if (LOG.isDebugEnabled()) { - LOG.debug("Write record {} {} {}", record.getKey(), record.getValue(), record.getSchema()); + try { + ConsumerRecords consumerRecords = consumer.poll(Duration.ofSeconds(1L)); + CompletableFuture[] futures = new CompletableFuture[consumerRecords.count()]; + int index = 0; + for (ConsumerRecord consumerRecord : consumerRecords) { + KafkaRecord record = buildRecord(consumerRecord); + if (LOG.isDebugEnabled()) { + LOG.debug("Write record {} {} {}", record.getKey(), record.getValue(), record.getSchema()); + } + consume(record); + futures[index] = record.getCompletableFuture(); + index++; } - consume(record); - futures[index] = record.getCompletableFuture(); - index++; - } - if (!kafkaSourceConfig.isAutoCommitEnabled()) { - try { + if (!kafkaSourceConfig.isAutoCommitEnabled()) { CompletableFuture.allOf(futures).get(); consumer.commitSync(); - } catch (InterruptedException ex) { - break; - } catch (ExecutionException ex) { - LOG.error("Error while processing records", ex); - break; } + } catch (Exception e) { + LOG.error("Error while processing records", e); + notifyError(e); + break; } } }); - runnerThread.setUncaughtExceptionHandler( - (t, e) -> { - new Thread(() -> { - LOG.error("[{}] Error while consuming records", t.getName(), e); - try { - this.close(); - } catch (Exception ex) { - LOG.error("[{}] Close kafka source error", t.getName(), e); - } - }, "Kafka Source Close Task Thread").start(); - }); + running = true; runnerThread.setName("Kafka Source Thread"); runnerThread.start(); } diff --git a/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaPushSource.java b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaPushSource.java new file mode 100644 index 0000000000000..d0d92daee8d5b --- /dev/null +++ b/pulsar-io/kafka/src/main/java/org/apache/pulsar/io/kafka/KafkaPushSource.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.kafka; + +import java.util.concurrent.LinkedBlockingQueue; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Source; + +/** + * Kafka Push Source. + * To maintain compatibility, we can't pick the PIP-281: https://github.com/apache/pulsar/pull/20807 + * cherry-pick to the historical version, so the class is implemented in the kafka connector. + */ +public abstract class KafkaPushSource implements Source { + + private static class NullRecord implements Record { + @Override + public Object getValue() { + return null; + } + } + + private static class ErrorNotifierRecord implements Record { + private Exception e; + public ErrorNotifierRecord(Exception e) { + this.e = e; + } + @Override + public Object getValue() { + return null; + } + + public Exception getException() { + return e; + } + } + + private LinkedBlockingQueue> queue; + private static final int DEFAULT_QUEUE_LENGTH = 1000; + private final NullRecord nullRecord = new NullRecord(); + + public KafkaPushSource() { + this.queue = new LinkedBlockingQueue<>(this.getQueueLength()); + } + + @Override + public Record read() throws Exception { + Record record = queue.take(); + if (record instanceof ErrorNotifierRecord) { + throw ((ErrorNotifierRecord) record).getException(); + } + if (record instanceof NullRecord) { + return null; + } else { + return record; + } + } + + /** + * Send this message to be written to Pulsar. + * Pass null if you you are done with this task + * @param record next message from source which should be sent to a Pulsar topic + */ + public void consume(Record record) { + try { + if (record != null) { + queue.put(record); + } else { + queue.put(nullRecord); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * Get length of the queue that records are push onto. + * Users can override this method to customize the queue length + * @return queue length + */ + public int getQueueLength() { + return DEFAULT_QUEUE_LENGTH; + } + + /** + * Allows the source to notify errors asynchronously. + * @param ex + */ + public void notifyError(Exception ex) { + consume(new ErrorNotifierRecord(ex)); + } +} diff --git a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml index f49ff86c90caf..47c3d73af5f06 100644 --- a/pulsar-io/kafka/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/kafka/src/main/resources/findbugsExclude.xml @@ -1,22 +1,23 @@ - - - \ No newline at end of file + + + + diff --git a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java index ee573cacd7719..54e7ac8f8354b 100644 --- a/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java +++ b/pulsar-io/kafka/src/test/java/org/apache/pulsar/io/kafka/source/KafkaAbstractSourceTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; +import java.time.Duration; import java.util.Collection; import java.lang.reflect.Field; import org.apache.kafka.clients.consumer.Consumer; @@ -31,7 +32,6 @@ import org.apache.pulsar.io.core.SourceContext; import org.apache.pulsar.io.kafka.KafkaAbstractSource; import org.apache.pulsar.io.kafka.KafkaSourceConfig; -import org.awaitility.Awaitility; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -158,26 +158,47 @@ public final void loadFromSaslYamlFileTest() throws IOException { assertEquals(config.getSslTruststorePassword(), "cert_pwd"); } - @Test - public final void closeConnectorWhenUnexpectedExceptionThrownTest() throws Exception { + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Subscribe exception") + public final void throwExceptionBySubscribe() throws Exception { KafkaAbstractSource source = new DummySource(); + + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); + Consumer consumer = mock(Consumer.class); - Mockito.doThrow(new RuntimeException("Uncaught exception")).when(consumer) + Mockito.doThrow(new RuntimeException("Subscribe exception")).when(consumer) .subscribe(Mockito.any(Collection.class)); Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); consumerField.setAccessible(true); consumerField.set(source, consumer); - + // will throw RuntimeException. source.start(); + } + + @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Pool exception") + public final void throwExceptionByPoll() throws Exception { + KafkaAbstractSource source = new DummySource(); - Field runningField = KafkaAbstractSource.class.getDeclaredField("running"); - runningField.setAccessible(true); + KafkaSourceConfig kafkaSourceConfig = new KafkaSourceConfig(); + kafkaSourceConfig.setTopic("test-topic"); + Field kafkaSourceConfigField = KafkaAbstractSource.class.getDeclaredField("kafkaSourceConfig"); + kafkaSourceConfigField.setAccessible(true); + kafkaSourceConfigField.set(source, kafkaSourceConfig); - Awaitility.await().untilAsserted(() -> { - Assert.assertFalse((boolean) runningField.get(source)); - Assert.assertNull(consumerField.get(source)); - }); + Consumer consumer = mock(Consumer.class); + Mockito.doThrow(new RuntimeException("Pool exception")).when(consumer) + .poll(Mockito.any(Duration.class)); + + Field consumerField = KafkaAbstractSource.class.getDeclaredField("consumer"); + consumerField.setAccessible(true); + consumerField.set(source, consumer); + source.start(); + // will throw RuntimeException. + source.read(); } private File getFile(String name) { diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 31aa0a61081ac..a98a3ba8579cb 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-kinesis Pulsar IO :: Kinesis - ${project.groupId} pulsar-io-common ${project.version} - ${project.groupId} pulsar-io-core ${project.version} - ${project.groupId} pulsar-functions-instance ${project.version} provided - ${project.groupId} pulsar-io-aws ${project.version} - org.apache.commons commons-lang3 - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.dataformat jackson-dataformat-cbor - com.google.code.gson gson - com.amazonaws aws-java-sdk-core - software.amazon.kinesis amazon-kinesis-client 2.2.8 - com.amazonaws amazon-kinesis-producer 0.14.0 - - + com.google.flatbuffers flatbuffers-java 1.9.0 - javax.xml.bind jaxb-api 2.3.0 - - @@ -123,5 +106,4 @@ - diff --git a/pulsar-io/mongo/pom.xml b/pulsar-io/mongo/pom.xml index 7e1e0772db028..8d3976ec20cb5 100644 --- a/pulsar-io/mongo/pom.xml +++ b/pulsar-io/mongo/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT + io.streamnative + pulsar-io + 2.10.5.3 - pulsar-io-mongo Pulsar IO :: MongoDB - 4.1.2 - ${project.parent.groupId} @@ -64,7 +59,6 @@ guava - diff --git a/pulsar-io/netty/pom.xml b/pulsar-io/netty/pom.xml index ce58ea295468c..4c6b3c1b4e75a 100644 --- a/pulsar-io/netty/pom.xml +++ b/pulsar-io/netty/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - org.apache.pulsar - pulsar-io - 2.10.5-SNAPSHOT - - - pulsar-io-netty - Pulsar IO :: Netty - - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - io.netty - netty-handler - - - - io.netty - netty-codec-http - - - - com.google.guava - guava - - - - org.apache.commons - commons-lang3 - - - - org.testng - testng - test - - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-io + 2.10.5.3 + + pulsar-io-netty + Pulsar IO :: Netty + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + io.netty + netty-handler + + + io.netty + netty-codec-http + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + org.testng + testng + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-io/netty/src/main/resources/findbugsExclude.xml b/pulsar-io/netty/src/main/resources/findbugsExclude.xml index 3a52062fc9233..6b8906a775bd3 100644 --- a/pulsar-io/netty/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/netty/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/pulsar-io/nsq/pom.xml b/pulsar-io/nsq/pom.xml index 80dcc2c61ead3..f6fbded9421f5 100644 --- a/pulsar-io/nsq/pom.xml +++ b/pulsar-io/nsq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-nsq Pulsar IO :: NSQ - ${project.groupId} pulsar-io-core ${project.version} - com.sproutsocial nsq-j ${nsq-client.version} - commons-collections commons-collections ${commons.collections.version} - org.apache.commons commons-lang3 3.4 - ${project.groupId} pulsar-io-common ${project.version} - - - - - org.apache.nifi - nifi-nar-maven-plugin - - + + + org.apache.nifi + nifi-nar-maven-plugin + + diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 24926c66b5890..8a5a0412f615d 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io Pulsar IO :: Parent - main @@ -75,7 +73,6 @@ nsq - core-modules @@ -89,7 +86,6 @@ - @@ -107,5 +103,4 @@ - diff --git a/pulsar-io/rabbitmq/pom.xml b/pulsar-io/rabbitmq/pom.xml index 08e2d96d68ec0..3915434b88181 100644 --- a/pulsar-io/rabbitmq/pom.xml +++ b/pulsar-io/rabbitmq/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-rabbitmq Pulsar IO :: RabbitMQ - - ${project.groupId} pulsar-io-core @@ -53,28 +50,23 @@ pulsar-client-original ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.google.guava guava - com.rabbitmq amqp-client ${rabbitmq-client.version} - org.apache.qpid qpid-broker @@ -86,9 +78,7 @@ awaitility test - - @@ -97,5 +87,4 @@ - diff --git a/pulsar-io/redis/pom.xml b/pulsar-io/redis/pom.xml index 26afd620196c1..ed1d47ddc8012 100644 --- a/pulsar-io/redis/pom.xml +++ b/pulsar-io/redis/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 2.10.5-SNAPSHOT - - - pulsar-io-redis - Pulsar IO :: Redis - - - - ${project.groupId} - pulsar-io-core - ${project.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - io.lettuce - lettuce-core - 5.0.2.RELEASE - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.apache.commons - commons-lang3 - 3.4 - - - commons-collections - commons-collections - 3.2.2 - - - com.github.kstyrc - embedded-redis - 0.6 - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 2.10.5.3 + + pulsar-io-redis + Pulsar IO :: Redis + + + ${project.groupId} + pulsar-io-core + ${project.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + io.lettuce + lettuce-core + 5.0.2.RELEASE + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.apache.commons + commons-lang3 + 3.4 + + + commons-collections + commons-collections + 3.2.2 + + + com.github.kstyrc + embedded-redis + 0.6 + test + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + + diff --git a/pulsar-io/solr/pom.xml b/pulsar-io/solr/pom.xml index 5c1788febb465..88e65f69954b4 100644 --- a/pulsar-io/solr/pom.xml +++ b/pulsar-io/solr/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - pulsar-io - org.apache.pulsar - 2.10.5-SNAPSHOT - - - - 8.11.1 - - - pulsar-io-solr - Pulsar IO :: Solr - - - - ${project.parent.groupId} - pulsar-io-core - ${project.parent.version} - - - ${project.groupId} - pulsar-functions-instance - ${project.version} - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - org.apache.solr - solr-solrj - ${solr.version} - - - org.apache.solr - solr-core - ${solr.version} - test - - - org.apache.commons - commons-lang3 - 3.4 - - - commons-collections - commons-collections - 3.2.2 - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - - - \ No newline at end of file + + 4.0.0 + + pulsar-io + io.streamnative + 2.10.5.3 + + + 8.11.1 + + pulsar-io-solr + Pulsar IO :: Solr + + + ${project.parent.groupId} + pulsar-io-core + ${project.parent.version} + + + ${project.groupId} + pulsar-functions-instance + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + org.apache.solr + solr-solrj + ${solr.version} + + + org.apache.solr + solr-core + ${solr.version} + test + + + org.apache.commons + commons-lang3 + 3.4 + + + commons-collections + commons-collections + 3.2.2 + + + + + + org.apache.nifi + nifi-nar-maven-plugin + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + spotbugs + verify + + check + + + + + + + diff --git a/pulsar-io/solr/src/main/resources/findbugsExclude.xml b/pulsar-io/solr/src/main/resources/findbugsExclude.xml index f49ff86c90caf..47c3d73af5f06 100644 --- a/pulsar-io/solr/src/main/resources/findbugsExclude.xml +++ b/pulsar-io/solr/src/main/resources/findbugsExclude.xml @@ -1,22 +1,23 @@ - - - \ No newline at end of file + + + + diff --git a/pulsar-io/solr/src/test/resources/solr.xml b/pulsar-io/solr/src/test/resources/solr.xml index 495a71f72f3f2..9ce4d37f4beef 100644 --- a/pulsar-io/solr/src/test/resources/solr.xml +++ b/pulsar-io/solr/src/test/resources/solr.xml @@ -1,3 +1,4 @@ + - - ${host:} - ${jetty.port:8983} - ${hostContext:solr} - ${genericCoreNodeNames:true} - ${zkClientTimeout:30000} - ${distribUpdateSoTimeout:600000} - ${distribUpdateConnTimeout:60000} - ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} - ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} - - - ${socketTimeout:600000} - ${connTimeout:60000} - - \ No newline at end of file + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${genericCoreNodeNames:true} + ${zkClientTimeout:30000} + ${distribUpdateSoTimeout:600000} + ${distribUpdateConnTimeout:60000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + + + ${socketTimeout:600000} + ${connTimeout:60000} + + diff --git a/pulsar-io/twitter/pom.xml b/pulsar-io/twitter/pom.xml index 1c51fd3ed7164..320f3a1fe5e74 100644 --- a/pulsar-io/twitter/pom.xml +++ b/pulsar-io/twitter/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar-io - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-io-twitter Pulsar IO :: Twitter - - ${project.groupId} pulsar-io-core ${project.version} - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.twitter hbc-core ${hbc-core.version} - commons-collections commons-collections ${commons.collections.version} - org.apache.commons commons-lang3 3.4 - - ${project.groupId} - pulsar-io-common - ${project.version} + ${project.groupId} + pulsar-io-common + ${project.version} - - @@ -82,5 +71,4 @@ - diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index 28444b0094dc3..6a81c9e633029 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-metadata Pulsar Metadata - org.apache.pulsar + io.streamnative pulsar-common ${project.version} - org.apache.zookeeper zookeeper - io.dropwizard.metrics metrics-core test - org.xerial.snappy snappy-java test - org.awaitility awaitility test - org.apache.bookkeeper bookkeeper-server - io.etcd jetcd-core - - io.etcd jetcd-test test - com.github.ben-manes.caffeine caffeine - - diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java index ef2df4980e982..c50521a889307 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataCache.java @@ -148,6 +148,11 @@ public interface MetadataCache { */ void invalidate(String path); + /** + * Force the invalidation of all object in the metadata cache. + */ + void invalidateAll(); + /** * Invalidate and reload an object in the metadata cache. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java index b4295f2586768..79670e859d6d0 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/api/MetadataStore.java @@ -50,6 +50,17 @@ public interface MetadataStore extends AutoCloseable { */ CompletableFuture> get(String path); + + /** + * Ensure that the next value read from the local client will be up-to-date with the latest version of the value + * as it can be seen by all the other clients. + * @param path + * @return a handle to the operation + */ + default CompletableFuture sync(String path) { + return CompletableFuture.completedFuture(null); + } + /** * Return all the nodes (lexicographically sorted) that are children to the specific path. * diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java index f9c33f2eb3559..0b4b53551194f 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java @@ -50,11 +50,13 @@ import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Notification; +import org.apache.pulsar.metadata.api.extended.SessionEvent; +import org.apache.pulsar.metadata.impl.AbstractMetadataStore; @Slf4j public class PulsarRegistrationClient implements RegistrationClient { - private final MetadataStore store; + private final AbstractMetadataStore store; private final String ledgersRootPath; // registration paths private final String bookieRegistrationPath; @@ -67,10 +69,11 @@ public class PulsarRegistrationClient implements RegistrationClient { private final Map> writableBookieInfo; private final Map> readOnlyBookieInfo; private final FutureUtil.Sequencer sequencer; + private SessionEvent lastMetadataSessionEvent; public PulsarRegistrationClient(MetadataStore store, String ledgersRootPath) { - this.store = store; + this.store = (AbstractMetadataStore) store; this.ledgersRootPath = ledgersRootPath; this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE); this.sequencer = Sequencer.create(); @@ -87,6 +90,7 @@ public PulsarRegistrationClient(MetadataStore store, .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-registration-client")); store.registerListener(this::updatedBookies); + this.store.registerSessionListener(this::refreshBookies); } @Override @@ -94,6 +98,21 @@ public void close() { executor.shutdownNow(); } + private void refreshBookies(SessionEvent sessionEvent) { + lastMetadataSessionEvent = sessionEvent; + if (!SessionEvent.Reconnected.equals(sessionEvent) && !SessionEvent.SessionReestablished.equals(sessionEvent)){ + return; + } + // Clean caches. + store.invalidateCaches(bookieRegistrationPath, bookieAllRegistrationPath, bookieReadonlyRegistrationPath); + bookieServiceInfoMetadataCache.invalidateAll(); + // Refresh caches of the listeners. + getReadOnlyBookies().thenAccept(bookies -> + readOnlyBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + getWritableBookies().thenAccept(bookies -> + writableBookiesWatchers.forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + } + @Override public CompletableFuture>> getWritableBookies() { return getBookiesThenFreshCache(bookieRegistrationPath); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java index c1845ba5f4fe6..97b8b5a305ac3 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/AbstractMetadataStore.java @@ -23,6 +23,7 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.annotations.VisibleForTesting; import io.netty.util.concurrent.DefaultThreadFactory; import java.util.EnumSet; @@ -317,6 +318,13 @@ public void invalidateAll() { existsCache.synchronous().invalidateAll(); } + public void invalidateCaches(String...paths) { + LoadingCache> loadingCache = childrenCache.synchronous(); + for (String path : paths) { + loadingCache.invalidate(path); + } + } + /** * Run the task in the executor thread and fail the future if the executor is shutting down. */ diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java index 6f980d3e79466..33e708a76b9dc 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/ZKMetadataStore.java @@ -153,6 +153,24 @@ protected void receivedSessionEvent(SessionEvent event) { } } + @Override + public CompletableFuture sync(String path) { + CompletableFuture result = new CompletableFuture<>(); + zkc.sync(path, new AsyncCallback.VoidCallback() { + @Override + public void processResult(int rc, String s, Object o) { + Code code = Code.get(rc); + if (code == Code.OK) { + result.complete(null); + } else { + MetadataStoreException e = getException(code, path); + result.completeExceptionally(e); + } + } + }, null); + return result; + } + @Override protected void batchOperation(List ops) { try { diff --git a/pulsar-metadata/src/test/resources/findbugsExclude.xml b/pulsar-metadata/src/test/resources/findbugsExclude.xml index e40b2a0fd9a12..bf075c39791cf 100644 --- a/pulsar-metadata/src/test/resources/findbugsExclude.xml +++ b/pulsar-metadata/src/test/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 4073cecd56392..faec255c55ded 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 2.10.5-SNAPSHOT - - 4.0.0 - - pulsar-package-bookkeeper-storage - Apache Pulsar :: Package Management :: BookKeeper Storage - - - - ${project.groupId} - pulsar-package-core - ${project.version} - - - - org.apache.distributedlog - distributedlog-core - - - net.jpountz.lz4 - lz4 - - - - - - org.apache.zookeeper - zookeeper - tests - test - - - - - io.dropwizard.metrics - metrics-core - test - - - - - org.xerial.snappy - snappy-java - test - - - - ${project.groupId} - managed-ledger - ${project.version} - test - - - - ${project.groupId} - testmocks - ${project.version} - - - - junit - junit - test - - - + + + pulsar-package-management + io.streamnative + 2.10.5.3 + + 4.0.0 + pulsar-package-bookkeeper-storage + Apache Pulsar :: Package Management :: BookKeeper Storage + + + ${project.groupId} + pulsar-package-core + ${project.version} + + + org.apache.distributedlog + distributedlog-core + + + net.jpountz.lz4 + lz4 + + + + + org.apache.zookeeper + zookeeper + tests + test + + + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + + ${project.groupId} + managed-ledger + ${project.version} + test + + + ${project.groupId} + testmocks + ${project.version} + + + junit + junit + test + + diff --git a/pulsar-package-management/core/pom.xml b/pulsar-package-management/core/pom.xml index c72d86753cde1..70d596ed598cb 100644 --- a/pulsar-package-management/core/pom.xml +++ b/pulsar-package-management/core/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar-package-management - org.apache.pulsar - 2.10.5-SNAPSHOT - - 4.0.0 - - pulsar-package-core - Apache Pulsar :: Package Management :: Core - - - - ${project.groupId} - pulsar-client-admin-api - ${project.parent.version} - - - - com.google.guava - guava - - - - com.google.code.gson - gson - - - - org.apache.commons - commons-lang3 - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - + + + pulsar-package-management + io.streamnative + 2.10.5.3 + + 4.0.0 + pulsar-package-core + Apache Pulsar :: Package Management :: Core + + + ${project.groupId} + pulsar-client-admin-api + ${project.parent.version} + + + com.google.guava + guava + + + com.google.code.gson + gson + + + org.apache.commons + commons-lang3 + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + diff --git a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-package-management/core/src/main/resources/findbugsExclude.xml +++ b/pulsar-package-management/core/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - pulsar-package-management - org.apache.pulsar - 2.10.5-SNAPSHOT - - 4.0.0 - - pulsar-package-filesystem-storage - Apache Pulsar :: Package Management :: Filesystem Storage - - - - ${project.groupId} - pulsar-package-core - ${project.parent.version} - - - - com.google.guava - guava - - - - ${project.groupId} - testmocks - ${project.parent.version} - test - - - - junit - junit - test - - - + + + pulsar-package-management + io.streamnative + 2.10.5.3 + + 4.0.0 + pulsar-package-filesystem-storage + Apache Pulsar :: Package Management :: Filesystem Storage + + + ${project.groupId} + pulsar-package-core + ${project.parent.version} + + + com.google.guava + guava + + + ${project.groupId} + testmocks + ${project.parent.version} + test + + + junit + junit + test + + diff --git a/pulsar-package-management/pom.xml b/pulsar-package-management/pom.xml index c7046e6278fa1..c612939432912 100644 --- a/pulsar-package-management/pom.xml +++ b/pulsar-package-management/pom.xml @@ -1,4 +1,4 @@ - + - - - pulsar - org.apache.pulsar - 2.10.5-SNAPSHOT - .. - - 4.0.0 - - pulsar-package-management - pom - Apache Pulsar :: Package Management - - core - bookkeeper-storage - filesystem-storage - - - - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - - spotbugs - verify - - check - - - - - - + + + pulsar + io.streamnative + 2.10.5.3 + .. + + 4.0.0 + pulsar-package-management + pom + Apache Pulsar :: Package Management + + core + bookkeeper-storage + filesystem-storage + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + spotbugs + verify + + check + + + + + + diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index 2920a5c0c7fc7..639138b8c3d30 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-proxy Pulsar Proxy - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-common ${project.version} - ${project.groupId} pulsar-websocket ${project.version} - org.apache.commons commons-lang3 - org.eclipse.jetty jetty-server - org.eclipse.jetty jetty-alpn-conscrypt-server - org.eclipse.jetty jetty-servlet - org.eclipse.jetty jetty-servlets - org.eclipse.jetty jetty-proxy - org.glassfish.jersey.core jersey-server - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.media jersey-media-json-jackson - com.google.guava guava - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.glassfish.jersey.inject jersey-hk2 - javax.xml.bind jaxb-api - com.sun.activation javax.activation - org.slf4j jul-to-slf4j - io.prometheus simpleclient - io.prometheus simpleclient_hotspot - io.prometheus simpleclient_servlet - io.prometheus simpleclient_jetty - ${project.groupId} pulsar-broker ${project.version} test - ${project.groupId} testmocks ${project.version} test - com.beust jcommander - org.apache.logging.log4j log4j-core - com.github.seancfoley ipaddress @@ -240,5 +211,4 @@ - diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index c28c7e635f0fb..b13d3e8b9e97a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -342,6 +342,12 @@ public class ProxyConfiguration implements PulsarConfiguration { + "to take effect" ) private boolean forwardAuthorizationCredentials = false; + @FieldContext( + category = CATEGORY_HTTP, + doc = "Whether to enable the proxy's /metrics, /proxy-stats, and /status.html http endpoints" + ) + private boolean enableProxyStatsEndpoints = true; + @FieldContext( category = CATEGORY_AUTHENTICATION, doc = "Whether the '/metrics' endpoint requires authentication. Defaults to true." diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java index d042323c74eb7..6bbe6691be3c8 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyServiceStarter.java @@ -241,17 +241,19 @@ public static void addWebServerHandlers(WebServer server, ProxyConfiguration config, ProxyService service, BrokerDiscoveryProvider discoveryProvider) throws Exception { - if (service != null) { - PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); - if (metricsServlet != null) { - server.addServlet("/metrics", new ServletHolder(metricsServlet), - Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + if (config.isEnableProxyStatsEndpoints()) { + server.addRestResources("/", VipStatus.class.getPackage().getName(), + VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath()); + server.addRestResources("/proxy-stats", ProxyStats.class.getPackage().getName(), + ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service); + if (service != null) { + PrometheusMetricsServlet metricsServlet = service.getMetricsServlet(); + if (metricsServlet != null) { + server.addServlet("/metrics", new ServletHolder(metricsServlet), + Collections.emptyList(), config.isAuthenticateMetricsEndpoint()); + } } } - server.addRestResources("/", VipStatus.class.getPackage().getName(), - VipStatus.ATTRIBUTE_STATUS_FILE_PATH, config.getStatusFilePath()); - server.addRestResources("/proxy-stats", ProxyStats.class.getPackage().getName(), - ProxyStats.ATTRIBUTE_PULSAR_PROXY_NAME, service); AdminProxyHandler adminProxyHandler = new AdminProxyHandler(config, discoveryProvider); ServletHolder servletHolder = new ServletHolder(adminProxyHandler); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index 35f717361f238..4269ac3d966fe 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -151,10 +151,18 @@ public void addServlet(String basePath, ServletHolder servletHolder, List> attributes, boolean requireAuthentication) { - Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); - if (existingPath.isPresent()) { - throw new IllegalArgumentException( - String.format("Cannot add servlet at %s, path %s already exists", basePath, existingPath.get())); + addServlet(basePath, servletHolder, attributes, requireAuthentication, true); + } + + private void addServlet(String basePath, ServletHolder servletHolder, + List> attributes, boolean requireAuthentication, boolean checkForExistingPaths) { + if (checkForExistingPaths) { + Optional existingPath = servletPaths.stream().filter(p -> p.startsWith(basePath)).findFirst(); + if (existingPath.isPresent()) { + throw new IllegalArgumentException( + String.format("Cannot add servlet at %s, path %s already exists", basePath, + existingPath.get())); + } } servletPaths.add(basePath); @@ -183,11 +191,9 @@ public void addRestResources(String basePath, String javaPackages, String attrib config.register(JsonMapperProvider.class); ServletHolder servletHolder = new ServletHolder(new ServletContainer(config)); servletHolder.setAsyncSupported(true); - ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); - context.setContextPath(basePath); - context.addServlet(servletHolder, "/*"); - context.setAttribute(attribute, attributeValue); - handlers.add(context); + // This method has not historically checked for existing paths, so we don't check here either. The + // method call is added to reduce code duplication. + addServlet(basePath, servletHolder, Collections.singletonList(Pair.of(attribute, attributeValue)), true, false); } public int getExternalServicePort() { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java index d0213899046c8..4a2bd92842f37 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/stats/ProxyStats.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.proxy.stats; +import static java.util.concurrent.TimeUnit.SECONDS; import io.netty.channel.Channel; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -27,7 +28,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -36,7 +40,12 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response.Status; +import org.apache.pulsar.broker.authentication.AuthenticationDataSource; +import org.apache.pulsar.broker.authentication.AuthenticationParameters; +import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.proxy.server.ProxyService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @@ -45,12 +54,15 @@ @Produces(MediaType.APPLICATION_JSON) public class ProxyStats { + private static final Logger log = LoggerFactory.getLogger(ProxyStats.class); public static final String ATTRIBUTE_PULSAR_PROXY_NAME = "pulsar-proxy"; private ProxyService service; @Context protected ServletContext servletContext; + @Context + protected HttpServletRequest httpRequest; @GET @Path("/connections") @@ -58,6 +70,7 @@ public class ProxyStats { response = List.class, responseContainer = "List") @ApiResponses(value = { @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public List metrics() { + throwIfNotSuperUser("metrics"); List stats = new ArrayList<>(); proxyService().getClientCnxs().forEach(cnx -> { if (cnx.getDirectProxyHandler() == null) { @@ -78,7 +91,7 @@ public List metrics() { @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy logging should be > 2 to capture topic stats"), @ApiResponse(code = 503, message = "Proxy service is not initialized") }) public Map topics() { - + throwIfNotSuperUser("topics"); Optional logLevel = proxyService().getConfiguration().getProxyLogLevel(); if (!logLevel.isPresent() || logLevel.get() < 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy doesn't have logging level 2"); @@ -92,6 +105,7 @@ public Map topics() { notes = "It only changes log-level in memory, change it config file to persist the change") @ApiResponses(value = { @ApiResponse(code = 412, message = "Proxy log level can be [0-2]"), }) public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("updateProxyLogLevel"); if (logLevel < 0 || logLevel > 2) { throw new RestException(Status.PRECONDITION_FAILED, "Proxy log level can be only [0-2]"); } @@ -102,6 +116,7 @@ public void updateProxyLogLevel(@PathParam("logLevel") int logLevel) { @Path("/logging") @ApiOperation(hidden = true, value = "Get proxy logging") public int getProxyLogLevel(@PathParam("logLevel") int logLevel) { + throwIfNotSuperUser("getProxyLogLevel"); return proxyService().getProxyLogLevel(); } @@ -114,4 +129,26 @@ protected ProxyService proxyService() { } return service; } + + private void throwIfNotSuperUser(String action) { + if (proxyService().getConfiguration().isAuthorizationEnabled()) { + AuthenticationParameters authParams = AuthenticationParameters.builder() + .clientRole((String) httpRequest.getAttribute(AuthenticationFilter.AuthenticatedRoleAttributeName)) + .clientAuthenticationDataSource((AuthenticationDataSource) + httpRequest.getAttribute(AuthenticationFilter.AuthenticatedDataAttributeName)) + .build(); + try { + if (authParams.getClientRole() == null + || !proxyService().getAuthorizationService().isSuperUser(authParams).get(30, SECONDS)) { + log.error("Client with role [{}] is not authorized to {}", authParams.getClientRole(), action); + throw new org.apache.pulsar.common.util.RestException(Status.UNAUTHORIZED, + "Client is not authorized to perform operation"); + } + } catch (ExecutionException | TimeoutException | InterruptedException e) { + log.warn("Time-out {} sec while checking the role {} is a super user role ", 30, + authParams.getClientRole()); + throw new org.apache.pulsar.common.util.RestException(Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + } } diff --git a/pulsar-sql/java-version-trim-agent/pom.xml b/pulsar-sql/java-version-trim-agent/pom.xml index 506375a2f8e91..12a7150554fda 100644 --- a/pulsar-sql/java-version-trim-agent/pom.xml +++ b/pulsar-sql/java-version-trim-agent/pom.xml @@ -1,3 +1,4 @@ + - - - pulsar-sql - org.apache.pulsar - 2.10.5-SNAPSHOT - - 4.0.0 - - java-version-trim-agent - Pulsar SQL :: Java Version Trim Agent - - - java-version-trim-agent - - - org.apache.maven.plugins - maven-jar-plugin - 3.1.0 - - - - true - - - org.apache.pulsar.sql.agent.TrimJavaVersionAgent - org.apache.pulsar.sql.agent.TrimJavaVersionAgent - true - true - - - - - - - - \ No newline at end of file + + + pulsar-sql + io.streamnative + 2.10.5.3 + + 4.0.0 + java-version-trim-agent + Pulsar SQL :: Java Version Trim Agent + + java-version-trim-agent + + + org.apache.maven.plugins + maven-jar-plugin + 3.1.0 + + + + true + + + org.apache.pulsar.sql.agent.TrimJavaVersionAgent + org.apache.pulsar.sql.agent.TrimJavaVersionAgent + true + true + + + + + + + diff --git a/pulsar-sql/pom.xml b/pulsar-sql/pom.xml index 188baa7f623ab..dc178e28f1779 100644 --- a/pulsar-sql/pom.xml +++ b/pulsar-sql/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - pom - - org.apache.pulsar - pulsar - 2.10.5-SNAPSHOT - - - pulsar-sql - Pulsar SQL :: Parent - - - presto-pulsar - presto-pulsar-plugin - java-version-trim-agent - presto-distribution - - - - - 3.14.9 - - 1.17.2 - 3.0.16 - - - - - - - com.squareup.okhttp3 - okhttp - ${okhttp3.version} - - - com.squareup.okhttp3 - okhttp-urlconnection - ${okhttp3.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp3.version} - - - com.squareup.okio - okio - ${okio.version} - - - - - org.slf4j - log4j-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-jdk14 - ${slf4j.version} - - - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - + 3.14.9 + + 1.17.2 + 3.0.16 + + + + + + com.squareup.okhttp3 + okhttp + ${okhttp3.version} + + + com.squareup.okhttp3 + okhttp-urlconnection + ${okhttp3.version} + + + com.squareup.okhttp3 + logging-interceptor + ${okhttp3.version} + + + com.squareup.okio + okio + ${okio.version} + + + + org.slf4j + log4j-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + + + - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index e97a4e720087d..6823ff06645d2 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar-sql - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-presto-distribution Pulsar SQL :: Pulsar Presto Distribution - false 2.34 @@ -44,7 +41,6 @@ 2.5.1 4.0.1 - org.glassfish.jersey.core @@ -76,7 +72,6 @@ jersey-client ${jersey.version} - org.codehaus.plexus plexus-utils @@ -110,13 +105,11 @@ - io.prestosql presto-cli ${presto.version} - io.airlift launcher @@ -124,7 +117,6 @@ tar.gz bin - io.airlift launcher @@ -132,75 +124,60 @@ tar.gz properties - org.objenesis objenesis ${objenesis.version} - com.twitter.common objectsize ${objectsize.version} - - com.fasterxml.jackson.core jackson-core - com.fasterxml.jackson.core jackson-databind - com.fasterxml.jackson.core jackson-annotations - com.fasterxml.jackson.datatype jackson-datatype-joda - com.fasterxml.jackson.dataformat jackson-dataformat-yaml - com.fasterxml.jackson.datatype jackson-datatype-guava - com.fasterxml.jackson.datatype jackson-datatype-jdk8 - com.fasterxml.jackson.datatype jackson-datatype-jsr310 - com.fasterxml.jackson.dataformat jackson-dataformat-smile - ${project.groupId} java-version-trim-agent ${project.version} provided - - @@ -213,7 +190,6 @@ netty 3.10.6.Final - org.apache.maven maven-core @@ -303,7 +279,6 @@ - @@ -313,7 +288,6 @@ ${skipBuildDistribution} - org.apache.maven.plugins maven-assembly-plugin @@ -337,7 +311,6 @@ - com.mycila license-maven-plugin @@ -362,7 +335,6 @@ - skipBuildDistributionDisabled diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml index bc1fe5ed46037..188295307f63b 100644 --- a/pulsar-sql/presto-distribution/src/assembly/assembly.xml +++ b/pulsar-sql/presto-distribution/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - false - - - ${basedir}/LICENSE - LICENSE - . - 644 - - - ${basedir}/src/main/resources/launcher.properties - launcher.properties - bin/ - 644 - - - ${basedir}/../java-version-trim-agent/target/java-version-trim-agent.jar - java-version-trim-agent.jar - / - - - - - ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ - plugin/ - - - - - lib/ - true - runtime - - io.airlift:launcher:tar.gz:bin:${airlift.version} - io.airlift:launcher:tar.gz:properties:${airlift.version} - *:tar.gz - - - - - - io.airlift:launcher:tar.gz:bin:${airlift.version} - - true - 755 - - - - - io.airlift:launcher:tar.gz:properties:${airlift.version} - - true - - - \ No newline at end of file + + bin + + tar.gz + dir + + false + + + ${basedir}/LICENSE + LICENSE + . + 644 + + + ${basedir}/src/main/resources/launcher.properties + launcher.properties + bin/ + 644 + + + ${basedir}/../java-version-trim-agent/target/java-version-trim-agent.jar + java-version-trim-agent.jar + / + + + + + ${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/ + plugin/ + + + + + lib/ + true + runtime + + io.airlift:launcher:tar.gz:bin:${airlift.version} + io.airlift:launcher:tar.gz:properties:${airlift.version} + *:tar.gz + + + + + + io.airlift:launcher:tar.gz:bin:${airlift.version} + + true + 755 + + + + + io.airlift:launcher:tar.gz:properties:${airlift.version} + + true + + + diff --git a/pulsar-sql/presto-pulsar-plugin/pom.xml b/pulsar-sql/presto-pulsar-plugin/pom.xml index cb99e8a197c5d..6e5c5b60e0035 100644 --- a/pulsar-sql/presto-pulsar-plugin/pom.xml +++ b/pulsar-sql/presto-pulsar-plugin/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 2.10.5-SNAPSHOT - - - pulsar-presto-connector - Pulsar SQL :: Pulsar Presto Connector - - - - - ${project.groupId} - pulsar-presto-connector-original - ${project.version} - - - - ${project.groupId} - bouncy-castle-bc - ${project.version} - pkg - true - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - 3.3.0 - - false - true - posix - - src/assembly/assembly.xml - - - - - package - package - - single - - - - - - - + + 4.0.0 + + io.streamnative + pulsar-sql + 2.10.5.3 + + pulsar-presto-connector + Pulsar SQL :: Pulsar Presto Connector + + + ${project.groupId} + pulsar-presto-connector-original + ${project.version} + + + ${project.groupId} + bouncy-castle-bc + ${project.version} + pkg + true + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + false + true + posix + + src/assembly/assembly.xml + + + + + package + package + + single + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + default-jar + package + + jar + + + + javadoc-jar + package + + jar + + + javadoc + + + + + + diff --git a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml index 6650abfda3fc3..649baf7318003 100644 --- a/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml +++ b/pulsar-sql/presto-pulsar-plugin/src/assembly/assembly.xml @@ -1,3 +1,4 @@ + - - bin - - tar.gz - dir - - - - / - false - runtime - - jakarta.ws.rs:jakarta.ws.rs-api - - - - \ No newline at end of file + + bin + + tar.gz + dir + + + + / + false + runtime + + jakarta.ws.rs:jakarta.ws.rs-api + + + + diff --git a/pulsar-sql/presto-pulsar/pom.xml b/pulsar-sql/presto-pulsar/pom.xml index 7145acaa1dbc5..ee779af8e858c 100644 --- a/pulsar-sql/presto-pulsar/pom.xml +++ b/pulsar-sql/presto-pulsar/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-sql - 2.10.5-SNAPSHOT - - - pulsar-presto-connector-original - Pulsar SQL - Pulsar Presto Connector - Pulsar SQL :: Pulsar Presto Connector Packaging - - - 0.199 - 2.1.2 - 1.8.4 - - - - - io.airlift - bootstrap - ${dep.airlift.version} - - - - io.airlift - json - ${dep.airlift.version} - - - - org.apache.avro - avro - ${avro.version} - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - org.jctools - jctools-core - ${jctools.version} - - - - com.dslplatform - dsl-json - ${dslJson.verson} - - - - io.prestosql - presto-plugin-toolkit - ${presto.version} - - - - - io.prestosql - presto-spi - ${presto.version} - provided - - - - joda-time - joda-time - ${joda.version} - - - - io.prestosql - presto-record-decoder - ${presto.version} - - - - javax.annotation - javax.annotation-api - ${javax.annotation-api.version} - - - - org.codehaus.plexus - plexus-utils - ${plexus.version} - - - io.prestosql - presto-main - ${presto.version} - test - - - org.codehaus.plexus - plexus-utils - - - - - - io.prestosql - presto-testing - ${presto.version} - test - - - - org.apache.pulsar + + 4.0.0 + + io.streamnative + pulsar-sql + 2.10.5.3 + + pulsar-presto-connector-original + Pulsar SQL - Pulsar Presto Connector + Pulsar SQL :: Pulsar Presto Connector Packaging + + 0.199 + 2.1.2 + 1.8.4 + + + + io.airlift + bootstrap + ${dep.airlift.version} + + + io.airlift + json + ${dep.airlift.version} + + + org.apache.avro + avro + ${avro.version} + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.jctools + jctools-core + ${jctools.version} + + + com.dslplatform + dsl-json + ${dslJson.verson} + + + io.prestosql + presto-plugin-toolkit + ${presto.version} + + + + io.prestosql + presto-spi + ${presto.version} + provided + + + joda-time + joda-time + ${joda.version} + + + io.prestosql + presto-record-decoder + ${presto.version} + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + org.codehaus.plexus + plexus-utils + ${plexus.version} + + + io.prestosql + presto-main + ${presto.version} + test + + + org.codehaus.plexus + plexus-utils + + + + + io.prestosql + presto-testing + ${presto.version} + test + + + io.streamnative + pulsar-broker + ${project.version} + test + + + io.streamnative + testmocks + ${project.version} + test + + + org.eclipse.jetty + jetty-http + ${jetty.version} + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + ${shadePluginPhase} + + shade + + + true + true + + + io.streamnative:pulsar-client-original + io.streamnative:pulsar-client-admin-original + io.streamnative:managed-ledger + io.streamnative:pulsar-metadata + org.glassfish.jersey*:* + javax.ws.rs:* + javax.annotation:* + org.glassfish.hk2*:* + org.eclipse.jetty:* + + + + + io.streamnative:pulsar-client-original + + ** + + + + org/bouncycastle/** + + + + + + org.glassfish + org.apache.pulsar.shade.org.glassfish + + + javax.ws + org.apache.pulsar.shade.javax.ws + + + javax.annotation + org.apache.pulsar.shade.javax.annotation + + + jersey + org.apache.pulsar.shade.jersey + + + org.eclipse.jetty + org.apache.pulsar.shade.org.eclipse.jetty + + + + + + + + + + + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} pulsar-broker ${project.version} + test-jar test - - - org.apache.pulsar - testmocks - ${project.version} - test - - - - org.eclipse.jetty - jetty-http - ${jetty.version} - test - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - ${shadePluginPhase} - - shade - - - true - true - - - - org.apache.pulsar:pulsar-client-original - org.apache.pulsar:pulsar-client-admin-original - org.apache.pulsar:managed-ledger - org.apache.pulsar:pulsar-metadata - - org.glassfish.jersey*:* - javax.ws.rs:* - javax.annotation:* - org.glassfish.hk2*:* - - org.eclipse.jetty:* - - - - - - org.apache.pulsar:pulsar-client-original - - ** - - - - org/bouncycastle/** - - - - - - org.glassfish - org.apache.pulsar.shade.org.glassfish - - - javax.ws - org.apache.pulsar.shade.javax.ws - - - javax.annotation - org.apache.pulsar.shade.javax.annotation - - - jersey - org.apache.pulsar.shade.jersey - - - org.eclipse.jetty - org.apache.pulsar.shade.org.eclipse.jetty - - - - - - - - - - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - + + + diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java index 827612f48be1c..01d27a2275498 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarRecordCursor.java @@ -767,7 +767,7 @@ private void initEntryCacheSizeAllocator(PulsarConnectorConfig connectorConfig) connectorConfig.getMaxSplitQueueSizeBytes() / 2); this.messageQueueCacheSizeAllocator = new NoStrictCacheSizeAllocator( connectorConfig.getMaxSplitQueueSizeBytes() / 2); - log.info("Init cacheSizeAllocator with maxSplitEntryQueueSizeBytes {}.", + log.info("Init cacheSizeAllocator with maxSplitEntryQueueSizeBytes %d.", connectorConfig.getMaxSplitQueueSizeBytes()); } else { this.entryQueueCacheSizeAllocator = new NullCacheSizeAllocator(); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java index 684ac5d2c7472..f4bcc2c348862 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarSqlSchemaInfoProvider.java @@ -71,7 +71,7 @@ public CompletableFuture getSchemaByVersion(byte[] schemaVersion) { } return completedFuture(cache.get(BytesSchemaVersion.of(schemaVersion))); } catch (ExecutionException e) { - LOG.error("Can't get generic schema for topic {} schema version {}", + LOG.error("Can't get generic schema for topic %s schema version %s", topicName.toString(), new String(schemaVersion, StandardCharsets.UTF_8), e); return FutureUtil.failedFuture(e.getCause()); } diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index ab39f2bd0010d..c1539e961b371 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar - pulsar - 2.10.5-SNAPSHOT - .. - - - pulsar-testclient - Pulsar Test Client - Pulsar Test Client - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.zookeeper - zookeeper - - - - ${project.groupId} - pulsar-client-admin-original - ${project.version} - - - - ${project.groupId} - pulsar-client-original - ${project.version} - - - - ${project.groupId} - pulsar-client-messagecrypto-bc - ${project.version} - true - - - - ${project.groupId} - pulsar-broker - ${project.version} - - - - commons-configuration - commons-configuration - - - - com.beust - jcommander - compile - - - - org.hdrhistogram - HdrHistogram - - - - com.fasterxml.jackson.core - jackson-databind - - - - org.awaitility - awaitility - test - - - - - - - - - test-jar-dependencies - - - maven.test.skip - !true - - - - - ${project.groupId} - pulsar-broker - ${project.version} - test-jar - test - - - - - - - - - org.gaul - modernizer-maven-plugin - - true - 8 - - - - modernizer - verify - - modernizer - - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - + + 4.0.0 + + io.streamnative + pulsar + 2.10.5.3 + .. + + pulsar-testclient + Pulsar Test Client + Pulsar Test Client + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.zookeeper + zookeeper + + + ${project.groupId} + pulsar-client-admin-original + ${project.version} + + + ${project.groupId} + pulsar-client-original + ${project.version} + + + ${project.groupId} + pulsar-client-messagecrypto-bc + ${project.version} + true + + + ${project.groupId} + pulsar-broker + ${project.version} + + + commons-configuration + commons-configuration + + + com.beust + jcommander + compile + + + org.hdrhistogram + HdrHistogram + + + com.fasterxml.jackson.core + jackson-databind + + + org.awaitility + awaitility + test + + + + + + test-jar-dependencies + + + maven.test.skip + !true + + + + + ${project.groupId} + pulsar-broker + ${project.version} + test-jar + test + + + + + + + + org.gaul + modernizer-maven-plugin + + true + 8 + + + + modernizer + verify + + modernizer + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + checkstyle + verify + + check + + + + + + diff --git a/pulsar-transaction/common/pom.xml b/pulsar-transaction/common/pom.xml index f42a4db0e43f4..3db383c128b3b 100644 --- a/pulsar-transaction/common/pom.xml +++ b/pulsar-transaction/common/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 2.10.5-SNAPSHOT - - - pulsar-transaction-common - Pulsar Transaction :: Common - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 2.10.5.3 + + pulsar-transaction-common + Pulsar Transaction :: Common + - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml index 07f4609cfff29..47c3d73af5f06 100644 --- a/pulsar-transaction/common/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/common/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - pulsar-transaction-parent - 2.10.5-SNAPSHOT - - - pulsar-transaction-coordinator - Pulsar Transaction :: Coordinator - - - - - ${project.groupId} - pulsar-common - ${project.version} - - - - ${project.groupId} - managed-ledger - ${project.version} - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.awaitility - awaitility - test - - - - - - - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - proto_path - ${project.parent.parent.basedir} - - - proto_search_strategy - 2 - - - - - - - - com.github.splunk.lightproto - lightproto-maven-plugin - ${lightproto-maven-plugin.version} - - - - generate - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - - ${basedir}/src/main/resources/findbugsExclude.xml - - - - + + 4.0.0 + + io.streamnative + pulsar-transaction-parent + 2.10.5.3 + + pulsar-transaction-coordinator + Pulsar Transaction :: Coordinator + + + ${project.groupId} + pulsar-common + ${project.version} + + + ${project.groupId} + managed-ledger + ${project.version} + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.awaitility + awaitility + test + + + + + + org.codehaus.mojo + properties-maven-plugin + + + initialize + + set-system-properties + + + + + proto_path + ${project.parent.parent.basedir} + + + proto_search_strategy + 2 + + + + + + + + com.github.splunk.lightproto + lightproto-maven-plugin + ${lightproto-maven-plugin.version} + + + + generate + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + ${basedir}/src/main/resources/findbugsExclude.xml + + + + diff --git a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml index 32fd17d259e15..b95f596528a6b 100644 --- a/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml +++ b/pulsar-transaction/coordinator/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - + + + + diff --git a/pulsar-transaction/pom.xml b/pulsar-transaction/pom.xml index 40b1a7c45d987..5529112c45563 100644 --- a/pulsar-transaction/pom.xml +++ b/pulsar-transaction/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - pulsar-transaction-parent Pulsar Transaction :: Parent - common coordinator - - - com.github.spotbugs spotbugs-maven-plugin @@ -68,7 +63,5 @@ - - diff --git a/pulsar-websocket/pom.xml b/pulsar-websocket/pom.xml index 9743c91c42be0..29cc089fff55d 100644 --- a/pulsar-websocket/pom.xml +++ b/pulsar-websocket/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - pulsar-websocket Pulsar WebSocket - ${project.groupId} pulsar-broker-common ${project.version} - ${project.groupId} pulsar-client-original ${project.version} - ${project.groupId} managed-ledger ${project.parent.version} test - org.apache.commons commons-lang3 - org.glassfish.jersey.containers jersey-container-servlet-core - org.glassfish.jersey.containers jersey-container-servlet - org.glassfish.jersey.inject jersey-hk2 - com.google.code.gson gson - io.swagger swagger-core - com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - org.slf4j jul-to-slf4j - - + org.eclipse.jetty.websocket websocket-api ${jetty.version} - - + org.eclipse.jetty.websocket websocket-server ${jetty.version} - + org.eclipse.jetty.websocket javax-websocket-client-impl @@ -115,9 +100,7 @@ org.hdrhistogram HdrHistogram - - @@ -137,7 +120,6 @@ - com.github.spotbugs spotbugs-maven-plugin @@ -155,7 +137,6 @@ - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/pulsar-websocket/src/main/resources/findbugsExclude.xml b/pulsar-websocket/src/main/resources/findbugsExclude.xml index 1121d71df9b9b..2cb607eed8253 100644 --- a/pulsar-websocket/src/main/resources/findbugsExclude.xml +++ b/pulsar-websocket/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + . - + true src/*.sh pulsar-client-cpp/docker/*.sh docker/pulsar/scripts/*.sh - data/** logs/** - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*] - - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?maven-eclipse\.xml] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.project] @@ -71,12 +67,10 @@ %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.externalToolBuilders(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.deployables(/.*)?] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?\.wtpmodules(/.*)?] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?cobertura\.ser] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.versionsBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?dependency-reduced-pom\.xml] - %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?pom\.xml\.releaseBackup] %regex[(?!((?!${project.build.directory}/)[^/]+/)*src/)(.*/)?release\.properties] @@ -85,7 +79,7 @@ ${project.build.directory}/maven-shared-archive-resources/META-INF - + src @@ -114,4 +108,4 @@ 0755 - \ No newline at end of file + diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh index afe380d5c8ffb..ec7e61a6fdee1 100755 --- a/src/check-binary-license.sh +++ b/src/check-binary-license.sh @@ -61,7 +61,7 @@ EXIT=0 # Check all bundled jars are mentioned in LICENSE for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi @@ -103,7 +103,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then for J in $JARS; do - echo $J | grep -q "org.apache.pulsar" + echo $J | grep -q "io.streamnative" if [ $? == 0 ]; then continue fi diff --git a/src/findbugs-exclude.xml b/src/findbugs-exclude.xml index 8f289b83a7be1..de4d3f7a93371 100644 --- a/src/findbugs-exclude.xml +++ b/src/findbugs-exclude.xml @@ -1,4 +1,4 @@ - + - - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/src/idea-code-style.xml b/src/idea-code-style.xml index 810dee34a3494..b49c523c322b2 100644 --- a/src/idea-code-style.xml +++ b/src/idea-code-style.xml @@ -1,3 +1,4 @@ + - - - - - \ No newline at end of file + diff --git a/src/owasp-dependency-check-false-positives.xml b/src/owasp-dependency-check-false-positives.xml index 013b95d747852..5e8014db31003 100644 --- a/src/owasp-dependency-check-false-positives.xml +++ b/src/owasp-dependency-check-false-positives.xml @@ -1,4 +1,4 @@ - + - file name: debezium-connector-postgres-1.7.2.Final.jar - ]]> + 69c1edfa7d89531af511fcd07e8516fa450f746a CVE-2021-23214 - - + - file name: mariadb-java-client-2.7.5.jar - ]]> + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 CVE-2022-27376 CVE-2022-27377 @@ -103,33 +101,28 @@ CVE-2022-27455 CVE-2022-27457 - - file name: google-http-client-gson-1.41.0.jar - ]]> + 1a754a5dd672218a2ac667d7ff2b28df7a5a240e CVE-2022-25647 - commons-net is not used at all and therefore commons-net vulnerability CVE-2021-37533 is a false positive. CVE-2021-37533 - fredsmith utils library is not used at all. CVE-2021-4277 is a false positive. CVE-2021-4277 - yaml_project is not used at all. Any CVEs reported for yaml_project are false positives. cpe:/a:yaml_project:yaml - flat_project is not used at all. cpe:/a:flat_project:flat - \ No newline at end of file + diff --git a/src/owasp-dependency-check-suppressions.xml b/src/owasp-dependency-check-suppressions.xml index e242ac52cec08..25c59252bf739 100644 --- a/src/owasp-dependency-check-suppressions.xml +++ b/src/owasp-dependency-check-suppressions.xml @@ -1,4 +1,4 @@ - + - - - Ignore netty CVEs in GRPC shaded Netty. - .*grpc-netty-shaded.* - cpe:/a:netty:netty - - - Suppress all pulsar-presto-distribution vulnerabilities - .*pulsar-presto-distribution.* - .* - - - Suppress libthrift-0.12.0.jar vulnerabilities - org.apache.thrift:libthrift:0.12.0 - .* - - - - + + Ignore netty CVEs in GRPC shaded Netty. + .*grpc-netty-shaded.* + cpe:/a:netty:netty + + + Suppress all pulsar-presto-distribution vulnerabilities + .*pulsar-presto-distribution.* + .* + + + Suppress libthrift-0.12.0.jar vulnerabilities + org.apache.thrift:libthrift:0.12.0 + .* + + + + file name: msgpack-core-0.9.0.jar - ]]> - 87d9ce0b22de48428fa32bb8ad476e18b6969548 - CVE-2022-41719 - - - - - Suppress Zookeeper 3.6.2 vulnerabilities - org\.apache\.zookeeper:.*:3\.6\.2 - .* - - - - - + 87d9ce0b22de48428fa32bb8ad476e18b6969548 + CVE-2022-41719 + + + + Suppress Zookeeper 3.6.2 vulnerabilities + org\.apache\.zookeeper:.*:3\.6\.2 + .* + + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2018-10237 - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2018-10237 + + + file name: canal.client-1.1.5.jar (shaded: com.google.guava:guava:22.0) - ]]> - b87878db57d5cfc2ca7d3972cc8f7486bf02fbca - CVE-2020-8908 - - - - - + b87878db57d5cfc2ca7d3972cc8f7486bf02fbca + CVE-2020-8908 + + + + file name: avro-1.10.2.jar - ]]> - a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 - CVE-2021-43045 - - - + a65aaa91c1aeceb3dd4859dbb9765d1c2063f5a2 + CVE-2021-43045 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14668 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14668 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14669 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14669 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14670 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14670 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14671 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14671 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2018-14672 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2018-14672 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-15024 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-15024 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-16535 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-16535 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2019-18657 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2019-18657 + + + file name: clickhouse-jdbc-0.3.2.jar - ]]> - fa9a1ccda7d78edb51a3a33d3493566092786a30 - CVE-2021-25263 - - - + fa9a1ccda7d78edb51a3a33d3493566092786a30 + CVE-2021-25263 + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - cpe:/a:qos:logback - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + cpe:/a:qos:logback + + + file name: rocketmq-acl-4.5.2.jar - ]]> - 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd - cpe:/a:apache:rocketmq - - - - ^pkg:maven/org\.springframework/spring.*$ - CVE-2016-1000027 - - - + 0e2bd9c162280cd79c2ea0f67f174ee5d7b84ddd + cpe:/a:apache:rocketmq + + + Ignored since we are not vulnerable + ^pkg:maven/org\.springframework/spring.*$ + CVE-2016-1000027 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - cpe:/a:qos:logback - - - + d90276fff414f06cb375f2057f6778cd63c6082f + cpe:/a:qos:logback + + + file name: logback-core-1.1.3.jar - ]]> - e3c02049f2dbbc764681b40094ecf0dcbc99b157 - CVE-2017-5929 - - - + e3c02049f2dbbc764681b40094ecf0dcbc99b157 + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2017-5929 - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2017-5929 + + + file name: logback-classic-1.1.3.jar - ]]> - d90276fff414f06cb375f2057f6778cd63c6082f - CVE-2021-42550 - - - - - + d90276fff414f06cb375f2057f6778cd63c6082f + CVE-2021-42550 + + + + file name: jetcd-core-0.5.11.jar - ]]> - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15106 - - - + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15106 + + + file name: jetcd-core-0.5.11.jar - ]]> - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15112 - - - + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15112 + + + file name: jetcd-core-0.5.11.jar - ]]> - c85851ca3ea8128d480d3f75c568a37e64e8a77b - CVE-2020-15113 - - - - + c85851ca3ea8128d480d3f75c568a37e64e8a77b + CVE-2020-15113 + + + file name: jetcd-common-0.5.11.jar - ]]> - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15106 - - - + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15106 + + + file name: jetcd-common-0.5.11.jar - ]]> - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15112 - - - + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15112 + + + file name: jetcd-common-0.5.11.jar - ]]> - 6dac6efe035a2be9ba299fbf31be5f903401869f - CVE-2020-15113 - - - - - + 6dac6efe035a2be9ba299fbf31be5f903401869f + CVE-2020-15113 + + + + file name: bc-fips-1.0.2.jar - ]]> - 4fb5db5f03d00f6a94e43b78d097978190e4abb2 - CVE-2020-26939 - - - + 4fb5db5f03d00f6a94e43b78d097978190e4abb2 + CVE-2020-26939 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-15522 - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-15522 + + + file name: bcpkix-fips-1.0.2.jar - ]]> - 543bc7a08cdba0172e95e536b5f7ca61f021253d - CVE-2020-26939 - - - - - + 543bc7a08cdba0172e95e536b5f7ca61f021253d + CVE-2020-26939 + + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2016-0738 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2016-0738 + + + file name: openstack-swift-2.5.0.jar - ]]> - d99d0eab2e01d69d8a326fc152427fbd759af88a - CVE-2017-16613 - - - + d99d0eab2e01d69d8a326fc152427fbd759af88a + CVE-2017-16613 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-14432 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-14432 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2018-20170 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2018-20170 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12689 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12689 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12690 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12690 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12691 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12691 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2020-12692 - - - + a7e89bd278fa8be9fa604dda66d1606de5530797 + CVE-2020-12692 + + + file name: openstack-keystone-2.5.0.jar - ]]> - a7e89bd278fa8be9fa604dda66d1606de5530797 - CVE-2021-3563 - - - - - + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:pulsar - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:pulsar + + + file name: org.apache.pulsar:pulsar-io-solr:2.10.0-SNAPSHOT - ]]> - ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ - cpe:/a:apache:solr - - - - - + ^pkg:maven/org\.apache\.pulsar/pulsar\-io\-solr@.*-SNAPSHOT$ + cpe:/a:apache:solr + + + + file name: debezium-connector-mysql-1.7.2.Final.jar - ]]> - a501bd758344d60fd400f5ce58694d52b2dbc6d8 - CVE-2010-1626 - CVE-2009-4028 - CVE-2007-1420 - CVE-2007-5925 - CVE-2007-2691 - CVE-2009-0819 - CVE-2010-1621 - CVE-2010-3677 - CVE-2010-3682 - CVE-2012-5627 - CVE-2015-2575 - CVE-2017-15945 - - - + a501bd758344d60fd400f5ce58694d52b2dbc6d8 + CVE-2010-1626 + CVE-2009-4028 + CVE-2007-1420 + CVE-2007-5925 + CVE-2007-2691 + CVE-2009-0819 + CVE-2010-1621 + CVE-2010-3677 + CVE-2010-3682 + CVE-2012-5627 + CVE-2015-2575 + CVE-2017-15945 + + + file name: mysql-binlog-connector-java-0.25.3.jar - ]]> - 45b3fdd0b953d744a8570f74eb5e1016f8ed5ca9 - CVE-2007-1420 - CVE-2007-2691 - CVE-2007-5925 - CVE-2009-0819 - CVE-2009-4028 - CVE-2010-1621 - CVE-2010-1626 - CVE-2010-3677 - CVE-2010-3682 - CVE-2012-5627 - CVE-2015-2575 - CVE-2017-15945 - - - + 45b3fdd0b953d744a8570f74eb5e1016f8ed5ca9 + CVE-2007-1420 + CVE-2007-2691 + CVE-2007-5925 + CVE-2009-0819 + CVE-2009-4028 + CVE-2010-1621 + CVE-2010-1626 + CVE-2010-3677 + CVE-2010-3682 + CVE-2012-5627 + CVE-2015-2575 + CVE-2017-15945 + + + file name: debezium-connector-postgres-1.7.2.Final.jar - ]]> - 69c1edfa7d89531af511fcd07e8516fa450f746a - CVE-2007-2138 - CVE-2010-0733 - CVE-2014-0060 - CVE-2014-0061 - CVE-2014-0062 - CVE-2014-0063 - CVE-2014-0064 - CVE-2014-0065 - CVE-2014-0066 - CVE-2014-0067 - CVE-2014-8161 - CVE-2015-0241 - CVE-2015-0242 - CVE-2015-0243 - CVE-2015-0244 - CVE-2015-3166 - CVE-2015-3167 - CVE-2016-0766 - CVE-2016-0768 - CVE-2016-0773 - CVE-2016-5423 - CVE-2016-5424 - CVE-2016-7048 - CVE-2017-14798 - CVE-2017-7484 - CVE-2018-1115 - CVE-2019-10127 - CVE-2019-10128 - CVE-2019-10210 - CVE-2019-10211 - CVE-2020-25694 - CVE-2020-25695 - CVE-2021-3393 - - - + 69c1edfa7d89531af511fcd07e8516fa450f746a + CVE-2007-2138 + CVE-2010-0733 + CVE-2014-0060 + CVE-2014-0061 + CVE-2014-0062 + CVE-2014-0063 + CVE-2014-0064 + CVE-2014-0065 + CVE-2014-0066 + CVE-2014-0067 + CVE-2014-8161 + CVE-2015-0241 + CVE-2015-0242 + CVE-2015-0243 + CVE-2015-0244 + CVE-2015-3166 + CVE-2015-3167 + CVE-2016-0766 + CVE-2016-0768 + CVE-2016-0773 + CVE-2016-5423 + CVE-2016-5424 + CVE-2016-7048 + CVE-2017-14798 + CVE-2017-7484 + CVE-2018-1115 + CVE-2019-10127 + CVE-2019-10128 + CVE-2019-10210 + CVE-2019-10211 + CVE-2020-25694 + CVE-2020-25695 + CVE-2021-3393 + + + file name: protostream-types-4.4.1.Final.jar - ]]> - 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 - CVE-2016-0750 - CVE-2017-15089 - CVE-2017-2638 - CVE-2019-10158 - CVE-2019-10174 - CVE-2020-25711 - - - + 29b45ebea1e4ce62ab3ec5eb76fa9771f98941b0 + CVE-2016-0750 + CVE-2017-15089 + CVE-2017-2638 + CVE-2019-10158 + CVE-2019-10174 + CVE-2020-25711 + + + file name: mariadb-java-client-2.7.5.jar - ]]> - 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 - CVE-2020-28912 - CVE-2021-46669 - CVE-2021-46666 - CVE-2021-46667 - - + + 9dd29797ecabe7d2e7fa892ec6713a5552cfcc59 + CVE-2020-28912 + CVE-2021-46669 + CVE-2021-46666 + CVE-2021-46667 + diff --git a/src/settings.xml b/src/settings.xml index 80ec2f40620e7..8fc2f13a37a59 100644 --- a/src/settings.xml +++ b/src/settings.xml @@ -1,3 +1,4 @@ + apache.releases.https @@ -34,7 +34,6 @@ ${env.APACHE_PASSWORD} - apache diff --git a/structured-event-log/pom.xml b/structured-event-log/pom.xml index b937463c99bca..dbe89f056cf1d 100644 --- a/structured-event-log/pom.xml +++ b/structured-event-log/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - structured-event-log Structured event logger - org.slf4j diff --git a/testmocks/pom.xml b/testmocks/pom.xml index a4aa179a9dcfa..cdfff01db7bec 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - pulsar - org.apache.pulsar - 2.10.5-SNAPSHOT + io.streamnative + 2.10.5.3 - testmocks jar Pulsar Test Mocks - - org.apache.zookeeper zookeeper - org.apache.bookkeeper bookkeeper-server - org.apache.commons commons-lang3 - org.testng testng - org.objenesis objenesis - org.powermock powermock-reflect - - @@ -83,5 +72,4 @@ - diff --git a/tests/bc_2_0_0/pom.xml b/tests/bc_2_0_0/pom.xml index 25b623fc8fe77..bfbb295e33d1d 100644 --- a/tests/bc_2_0_0/pom.xml +++ b/tests/bc_2_0_0/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - - bc_2_0_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.0-rc1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 2.10.5.3 + + bc_2_0_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.0-rc1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.0-rc1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_0/src/test/resources/pulsar.xml b/tests/bc_2_0_0/src/test/resources/pulsar.xml index 4b2a9bcb5dabb..cd21863f47d8c 100644 --- a/tests/bc_2_0_0/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_0/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_0_1/pom.xml b/tests/bc_2_0_1/pom.xml index 43c1de4b5cf19..15fbf6a15e88d 100644 --- a/tests/bc_2_0_1/pom.xml +++ b/tests/bc_2_0_1/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - - bc_2_0_1 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating - - - - - org.apache.pulsar - pulsar-client - 2.0.1-incubating - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 2.10.5.3 + + bc_2_0_1 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.0.1-incubating + + + org.apache.pulsar + pulsar-client + 2.0.1-incubating + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/bc_2_0_1/src/test/resources/pulsar.xml b/tests/bc_2_0_1/src/test/resources/pulsar.xml index 4b2a9bcb5dabb..cd21863f47d8c 100644 --- a/tests/bc_2_0_1/src/test/resources/pulsar.xml +++ b/tests/bc_2_0_1/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/bc_2_6_0/pom.xml b/tests/bc_2_6_0/pom.xml index 5a0694c037d41..a8893aa9948e2 100644 --- a/tests/bc_2_6_0/pom.xml +++ b/tests/bc_2_6_0/pom.xml @@ -1,4 +1,4 @@ - + - - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - 4.0.0 - - bc_2_6_0 - jar - Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 - - - - - com.google.code.gson - gson - test - - - - org.apache.pulsar - pulsar-client - 2.6.0 - test - - - - org.testcontainers - testcontainers - test - - - - - + + + io.streamnative.tests + tests-parent + 2.10.5.3 + + 4.0.0 + bc_2_6_0 + jar + Apache Pulsar :: Tests :: Backwards Client Compatibility 2.6.0 + + + com.google.code.gson + gson + test + + + org.apache.pulsar + pulsar-client + 2.6.0 + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + BackwardsCompatTests + + + BackwardsCompatTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - BackwardsCompatTests - - - BackwardsCompatTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/backwards-client.xml - - 1 - - - - - - - - + false + + src/test/resources/backwards-client.xml + + 1 + + + + + + diff --git a/tests/bc_2_6_0/src/test/resources/backwards-client.xml b/tests/bc_2_6_0/src/test/resources/backwards-client.xml index 4b2a9bcb5dabb..cd21863f47d8c 100644 --- a/tests/bc_2_6_0/src/test/resources/backwards-client.xml +++ b/tests/bc_2_6_0/src/test/resources/backwards-client.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/docker-images/java-test-functions/pom.xml b/tests/docker-images/java-test-functions/pom.xml index 0337e26bf7d6d..2dcfc2e952569 100644 --- a/tests/docker-images/java-test-functions/pom.xml +++ b/tests/docker-images/java-test-functions/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 2.10.5-SNAPSHOT + 2.10.5.3 4.0.0 java-test-functions Apache Pulsar :: Tests :: Docker Images :: Java Test Functions - org.apache.pulsar + io.streamnative pulsar-io-core ${project.version} - org.apache.pulsar + io.streamnative pulsar-functions-api ${project.version} @@ -53,7 +53,6 @@ jar - docker @@ -62,15 +61,13 @@ integrationTests - - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} - @@ -85,7 +82,7 @@ - org.apache.pulsar:pulsar-functions-api-examples + io.streamnative:pulsar-functions-api-examples @@ -96,5 +93,4 @@ - diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index 994b12b4d2fb4..2dc753d6408fb 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 2.10.5-SNAPSHOT + 2.10.5.3 4.0.0 java-test-image Apache Pulsar :: Tests :: Docker Images :: Java Test Image pom - docker @@ -40,12 +39,12 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin @@ -73,7 +72,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar @@ -82,7 +81,7 @@ java-test-functions.jar - org.apache.pulsar + io.streamnative pulsar-server-distribution ${project.parent.version} bin diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 7918d557df047..5f30924559932 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -1,3 +1,4 @@ + - + - org.apache.pulsar.tests + io.streamnative.tests docker-images - 2.10.5-SNAPSHOT + 2.10.5.3 4.0.0 latest-version-image Apache Pulsar :: Tests :: Docker Images :: Latest Version Testing pom - docker @@ -40,12 +39,12 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} - org.apache.pulsar + io.streamnative pulsar-all-docker-image ${project.parent.version} docker-info @@ -65,7 +64,7 @@ - org.apache.pulsar.tests + io.streamnative.tests java-test-functions ${project.parent.version} jar diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 82dd850af0728..85f523e199dd9 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent - 2.10.5-SNAPSHOT + 2.10.5.3 docker-images Apache Pulsar :: Tests :: Docker Images diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index c456c3e7b4f25..701373f272b71 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -1,4 +1,4 @@ - + - + 4.0.0 - org.apache.pulsar.tests + io.streamnative.tests tests-parent - 2.10.5-SNAPSHOT + 2.10.5.3 - integration jar Apache Pulsar :: Tests :: Integration - pulsar.xml - com.google.code.gson @@ -43,37 +39,37 @@ test - org.apache.pulsar + io.streamnative pulsar-functions-api-examples ${project.version} test - org.apache.pulsar + io.streamnative pulsar-broker ${project.version} test - org.apache.pulsar + io.streamnative pulsar-common ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client ${project.version} test - org.apache.pulsar + io.streamnative pulsar-client-admin ${project.version} test - org.apache.pulsar + io.streamnative managed-ledger ${project.version} test @@ -95,89 +91,76 @@ test - org.apache.pulsar + io.streamnative pulsar-io-kafka ${project.version} test - org.testcontainers mysql test - org.postgresql postgresql ${postgresql-jdbc.version} runtime - org.testcontainers postgresql test - com.github.docker-java docker-java-core test - - org.apache.pulsar + io.streamnative pulsar-io-jdbc-postgres ${project.version} test - com.fasterxml.jackson.core jackson-databind test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml test - org.opensearch.client opensearch-rest-high-level-client test - - + com.rabbitmq amqp-client ${rabbitmq-client.version} test - joda-time joda-time test - io.prestosql presto-jdbc ${presto.version} test - org.awaitility awaitility test - @@ -218,7 +201,6 @@ - integrationTests diff --git a/tests/integration/src/test/resources/pulsar-auth.xml b/tests/integration/src/test/resources/pulsar-auth.xml index 81d2c1361a269..3647730dee201 100644 --- a/tests/integration/src/test/resources/pulsar-auth.xml +++ b/tests/integration/src/test/resources/pulsar-auth.xml @@ -1,3 +1,4 @@ + - + - - + + - + diff --git a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml index b50af74ed17f0..b0f55f3e04b16 100644 --- a/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml +++ b/tests/integration/src/test/resources/pulsar-backwards-compatibility.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-cli-suite.xml b/tests/integration/src/test/resources/pulsar-cli-suite.xml index ec07208a85893..51e95e5362938 100644 --- a/tests/integration/src/test/resources/pulsar-cli-suite.xml +++ b/tests/integration/src/test/resources/pulsar-cli-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/pulsar-cli.xml b/tests/integration/src/test/resources/pulsar-cli.xml index 70b4136f2ca76..cab0c733de49f 100644 --- a/tests/integration/src/test/resources/pulsar-cli.xml +++ b/tests/integration/src/test/resources/pulsar-cli.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-function.xml b/tests/integration/src/test/resources/pulsar-function.xml index a18a65ff2e16d..03a8d83997a3c 100644 --- a/tests/integration/src/test/resources/pulsar-function.xml +++ b/tests/integration/src/test/resources/pulsar-function.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-ora-source.xml b/tests/integration/src/test/resources/pulsar-io-ora-source.xml index 1c5bb5faf677c..0a9927015be86 100644 --- a/tests/integration/src/test/resources/pulsar-io-ora-source.xml +++ b/tests/integration/src/test/resources/pulsar-io-ora-source.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sinks.xml b/tests/integration/src/test/resources/pulsar-io-sinks.xml index 614c371f3e52c..046f977f3e13b 100644 --- a/tests/integration/src/test/resources/pulsar-io-sinks.xml +++ b/tests/integration/src/test/resources/pulsar-io-sinks.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-sources.xml b/tests/integration/src/test/resources/pulsar-io-sources.xml index 636b3e479195f..c995a7df249dd 100644 --- a/tests/integration/src/test/resources/pulsar-io-sources.xml +++ b/tests/integration/src/test/resources/pulsar-io-sources.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - \ No newline at end of file + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-io-suite.xml b/tests/integration/src/test/resources/pulsar-io-suite.xml index 64db138b28279..3718f66a8ca89 100644 --- a/tests/integration/src/test/resources/pulsar-io-suite.xml +++ b/tests/integration/src/test/resources/pulsar-io-suite.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-messaging.xml b/tests/integration/src/test/resources/pulsar-messaging.xml index cfbdb22587034..f8813b614f5cb 100644 --- a/tests/integration/src/test/resources/pulsar-messaging.xml +++ b/tests/integration/src/test/resources/pulsar-messaging.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-process-suite.xml b/tests/integration/src/test/resources/pulsar-process-suite.xml index 5c65e2d7eeb4d..94b72f2f6b49d 100644 --- a/tests/integration/src/test/resources/pulsar-process-suite.xml +++ b/tests/integration/src/test/resources/pulsar-process-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/pulsar-process.xml b/tests/integration/src/test/resources/pulsar-process.xml index 8e5258d30624c..a6f1ac468d954 100644 --- a/tests/integration/src/test/resources/pulsar-process.xml +++ b/tests/integration/src/test/resources/pulsar-process.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-proxy-websocket.xml b/tests/integration/src/test/resources/pulsar-proxy-websocket.xml index 6e32ac9f454a0..26e5fa0f9cf38 100644 --- a/tests/integration/src/test/resources/pulsar-proxy-websocket.xml +++ b/tests/integration/src/test/resources/pulsar-proxy-websocket.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-proxy.xml b/tests/integration/src/test/resources/pulsar-proxy.xml index ae6f13810535c..fabfb54d781a3 100644 --- a/tests/integration/src/test/resources/pulsar-proxy.xml +++ b/tests/integration/src/test/resources/pulsar-proxy.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-python.xml b/tests/integration/src/test/resources/pulsar-python.xml index a5faa6389e0f1..365768083d21b 100644 --- a/tests/integration/src/test/resources/pulsar-python.xml +++ b/tests/integration/src/test/resources/pulsar-python.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-schema-suite.xml b/tests/integration/src/test/resources/pulsar-schema-suite.xml index ed0a003a0bb90..dbfb5c063ea86 100644 --- a/tests/integration/src/test/resources/pulsar-schema-suite.xml +++ b/tests/integration/src/test/resources/pulsar-schema-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/pulsar-schema.xml b/tests/integration/src/test/resources/pulsar-schema.xml index e07fdf2b2d86f..595bb8fda5521 100644 --- a/tests/integration/src/test/resources/pulsar-schema.xml +++ b/tests/integration/src/test/resources/pulsar-schema.xml @@ -1,3 +1,4 @@ + - + - - - - - - - + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-semantics.xml b/tests/integration/src/test/resources/pulsar-semantics.xml index 5b5402af4623b..69e171931f0e4 100644 --- a/tests/integration/src/test/resources/pulsar-semantics.xml +++ b/tests/integration/src/test/resources/pulsar-semantics.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-sql-suite.xml b/tests/integration/src/test/resources/pulsar-sql-suite.xml index 78b6268bc3493..e0f1e47285312 100644 --- a/tests/integration/src/test/resources/pulsar-sql-suite.xml +++ b/tests/integration/src/test/resources/pulsar-sql-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/pulsar-sql.xml b/tests/integration/src/test/resources/pulsar-sql.xml index bedc443f6c9b6..a9c3b73aeeb5b 100644 --- a/tests/integration/src/test/resources/pulsar-sql.xml +++ b/tests/integration/src/test/resources/pulsar-sql.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-standalone.xml b/tests/integration/src/test/resources/pulsar-standalone.xml index d8892c0746181..45d79a72821b5 100644 --- a/tests/integration/src/test/resources/pulsar-standalone.xml +++ b/tests/integration/src/test/resources/pulsar-standalone.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-thread-suite.xml b/tests/integration/src/test/resources/pulsar-thread-suite.xml index 00f7220b7cf8a..b75c00ab4c455 100644 --- a/tests/integration/src/test/resources/pulsar-thread-suite.xml +++ b/tests/integration/src/test/resources/pulsar-thread-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/pulsar-thread.xml b/tests/integration/src/test/resources/pulsar-thread.xml index cf3da15e8e1ad..dc1b8e3b3de15 100644 --- a/tests/integration/src/test/resources/pulsar-thread.xml +++ b/tests/integration/src/test/resources/pulsar-thread.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - \ No newline at end of file + + + + + + + + + diff --git a/tests/integration/src/test/resources/pulsar-transaction.xml b/tests/integration/src/test/resources/pulsar-transaction.xml index 72c375d000d8e..425f58ebda639 100644 --- a/tests/integration/src/test/resources/pulsar-transaction.xml +++ b/tests/integration/src/test/resources/pulsar-transaction.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar-upgrade.xml b/tests/integration/src/test/resources/pulsar-upgrade.xml index a52db54753372..cb312b04ed7a3 100644 --- a/tests/integration/src/test/resources/pulsar-upgrade.xml +++ b/tests/integration/src/test/resources/pulsar-upgrade.xml @@ -1,3 +1,4 @@ + - + - - - - - + + + + + diff --git a/tests/integration/src/test/resources/pulsar.xml b/tests/integration/src/test/resources/pulsar.xml index 5382d9b5f5bef..bf4de78a3b724 100644 --- a/tests/integration/src/test/resources/pulsar.xml +++ b/tests/integration/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + diff --git a/tests/integration/src/test/resources/tiered-file-system-suite.xml b/tests/integration/src/test/resources/tiered-file-system-suite.xml index 46c2c3587d3b5..641403084ae64 100644 --- a/tests/integration/src/test/resources/tiered-file-system-suite.xml +++ b/tests/integration/src/test/resources/tiered-file-system-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/integration/src/test/resources/tiered-filesystem-storage.xml b/tests/integration/src/test/resources/tiered-filesystem-storage.xml index b14077594e0c9..c8cb694ea2d02 100644 --- a/tests/integration/src/test/resources/tiered-filesystem-storage.xml +++ b/tests/integration/src/test/resources/tiered-filesystem-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/tiered-jcloud-storage.xml b/tests/integration/src/test/resources/tiered-jcloud-storage.xml index f61f9f5fa1612..be699ba0a8680 100644 --- a/tests/integration/src/test/resources/tiered-jcloud-storage.xml +++ b/tests/integration/src/test/resources/tiered-jcloud-storage.xml @@ -1,3 +1,4 @@ + - + - - - - - - \ No newline at end of file + + + + + + diff --git a/tests/integration/src/test/resources/tiered-storage-jcloud-suite.xml b/tests/integration/src/test/resources/tiered-storage-jcloud-suite.xml index be7fa982fb715..e57ce1557d0c2 100644 --- a/tests/integration/src/test/resources/tiered-storage-jcloud-suite.xml +++ b/tests/integration/src/test/resources/tiered-storage-jcloud-suite.xml @@ -1,3 +1,4 @@ + - + - - - + + + diff --git a/tests/pom.xml b/tests/pom.xml index 700c9f4c92092..165cf406027bc 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -1,4 +1,4 @@ - + - + pom 4.0.0 - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 - org.apache.pulsar.tests + io.streamnative.tests tests-parent Apache Pulsar :: Tests @@ -50,7 +49,7 @@ skipIntegrationTests - + integrationTests diff --git a/tests/pulsar-client-admin-shade-test/pom.xml b/tests/pulsar-client-admin-shade-test/pom.xml index 0972471d11471..5039886454529 100644 --- a/tests/pulsar-client-admin-shade-test/pom.xml +++ b/tests/pulsar-client-admin-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - - pulsar-client-admin-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test - - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin-api - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 2.10.5.3 + + pulsar-client-admin-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Admin-Shade Test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + io.streamnative + pulsar-client-admin-api + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml index e32e9b464a15a..7c8bf2d74c577 100644 --- a/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-admin-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-all-shade-test/pom.xml b/tests/pulsar-client-all-shade-test/pom.xml index 03c2f4436e35c..59f118e2c1945 100644 --- a/tests/pulsar-client-all-shade-test/pom.xml +++ b/tests/pulsar-client-all-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - - pulsar-client-all-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test - - - - - org.apache.pulsar - pulsar-client-all - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - org.apache.pulsar - bouncy-castle-bc - ${project.version} - pkg - - - - org.apache.pulsar - pulsar-client-messagecrypto-bc - ${project.version} - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 2.10.5.3 + + pulsar-client-all-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-All-Shade Test + + + io.streamnative + pulsar-client-all + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + io.streamnative + bouncy-castle-bc + ${project.version} + pkg + + + io.streamnative + pulsar-client-messagecrypto-bc + ${project.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml index 746cc3d1b9e33..c4fc288600bae 100644 --- a/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-all-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tests/pulsar-client-shade-test/pom.xml b/tests/pulsar-client-shade-test/pom.xml index af22ca795e109..75072cab7df4e 100644 --- a/tests/pulsar-client-shade-test/pom.xml +++ b/tests/pulsar-client-shade-test/pom.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 - - - org.apache.pulsar.tests - tests-parent - 2.10.5-SNAPSHOT - - - pulsar-client-shade-test - jar - Apache Pulsar :: Tests :: Pulsar-Client-Shade Test - - - - - org.apache.pulsar - pulsar-client - ${project.version} - test - - - - org.apache.pulsar - pulsar-client-admin - ${project.version} - test - - - - org.testcontainers - testcontainers - test - - - - - + + 4.0.0 + + io.streamnative.tests + tests-parent + 2.10.5.3 + + pulsar-client-shade-test + jar + Apache Pulsar :: Tests :: Pulsar-Client-Shade Test + + + io.streamnative + pulsar-client + ${project.version} + test + + + io.streamnative + pulsar-client-admin + ${project.version} + test + + + org.testcontainers + testcontainers + test + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + true + + ${project.version} + ${project.build.directory} + + + + + + + + ShadeTests + + + ShadeTests + + + - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - true - - ${project.version} - ${project.build.directory} - - - - - - - - - ShadeTests - - - ShadeTests - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G + + org.apache.maven.plugins + maven-surefire-plugin + + ${testJacocoAgentArgument} -XX:+ExitOnOutOfMemoryError -Xmx2G -XX:MaxDirectMemorySize=8G -Dio.netty.leakDetectionLevel=advanced ${test.additional.args} - false - - src/test/resources/pulsar.xml - - 1 - - - - - - + false + + src/test/resources/pulsar.xml + + 1 + + + + + + diff --git a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml index df07642a01851..b83f6d760bb22 100644 --- a/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml +++ b/tests/pulsar-client-shade-test/src/test/resources/pulsar.xml @@ -1,3 +1,4 @@ + - + - - - - - - + + + + + + diff --git a/tiered-storage/file-system/pom.xml b/tiered-storage/file-system/pom.xml index 9739a04ece167..90a74d72594e2 100644 --- a/tiered-storage/file-system/pom.xml +++ b/tiered-storage/file-system/pom.xml @@ -1,3 +1,4 @@ + - - 4.0.0 - - - org.apache.pulsar - tiered-storage-parent - 2.10.5-SNAPSHOT - .. - - - tiered-storage-file-system - Apache Pulsar :: Tiered Storage :: File System - - - ${project.groupId} - managed-ledger - ${project.version} - - - org.apache.hadoop - hadoop-common - ${hdfs-offload-version3} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hadoop - hadoop-hdfs-client - ${hdfs-offload-version3} - - - org.apache.avro - avro - - - org.mortbay.jetty - jetty - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - - - - org.apache.avro - avro - ${avro.version} - - - - net.minidev - json-smart - ${json-smart.version} - - - com.google.protobuf - protobuf-java - - - - ${project.groupId} - testmocks - ${project.version} - test - - - - org.apache.hadoop - hadoop-minicluster - ${hdfs-offload-version3} - test - - - io.netty - netty-all - - - - - + + 4.0.0 + + io.streamnative + tiered-storage-parent + 2.10.5.3 + .. + + tiered-storage-file-system + Apache Pulsar :: Tiered Storage :: File System + + + ${project.groupId} + managed-ledger + ${project.version} + + + org.apache.hadoop + hadoop-common + ${hdfs-offload-version3} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hdfs-offload-version3} + + + org.apache.avro + avro + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + javax.servlet + servlet-api + + + + + + org.apache.avro + avro + ${avro.version} + + + + net.minidev + json-smart + ${json-smart.version} + + + com.google.protobuf + protobuf-java + + + ${project.groupId} + testmocks + ${project.version} + test + + + org.apache.hadoop + hadoop-minicluster + ${hdfs-offload-version3} + test + + io.netty - netty-codec-http - - - - org.eclipse.jetty - jetty-server - test - - - org.eclipse.jetty - jetty-alpn-conscrypt-server - test - - - org.eclipse.jetty - jetty-servlet - test - - - org.eclipse.jetty - jetty-util - test - - - - - - - org.apache.nifi - nifi-nar-maven-plugin - - - com.github.spotbugs - spotbugs-maven-plugin - ${spotbugs-maven-plugin.version} - - ${basedir}/../src/main/resources/findbugsExclude.xml - - - - spotbugs - verify - - check - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - checkstyle - verify - - check - - - - - - - - - - owasp-dependency-check - - - - org.owasp - dependency-check-maven - ${dependency-check-maven.version} - - - - aggregate - - none - - - - - - - + + owasp-dependency-check + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + + aggregate + + none + + + + + + + diff --git a/tiered-storage/jcloud/pom.xml b/tiered-storage/jcloud/pom.xml index 65e777fcf615b..a182ffa608400 100644 --- a/tiered-storage/jcloud/pom.xml +++ b/tiered-storage/jcloud/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 - - org.apache.pulsar + io.streamnative tiered-storage-parent - 2.10.5-SNAPSHOT + 2.10.5.3 .. - tiered-storage-jcloud Apache Pulsar :: Tiered Storage :: JCloud - ${project.groupId} managed-ledger ${project.version} - ${project.groupId} jclouds-shaded @@ -70,7 +66,6 @@ - org.apache.jclouds jclouds-allblobstore @@ -81,12 +76,10 @@ com.amazonaws aws-java-sdk-core - com.amazonaws aws-java-sdk-sts - ${project.groupId} testmocks @@ -99,7 +92,6 @@ ${jclouds.version} provided - javax.xml.bind jaxb-api @@ -111,13 +103,11 @@ runtime - com.sun.activation javax.activation runtime - @@ -125,7 +115,6 @@ org.apache.nifi nifi-nar-maven-plugin - com.github.spotbugs spotbugs-maven-plugin diff --git a/tiered-storage/pom.xml b/tiered-storage/pom.xml index 3178ff483b59e..7116dbdc65321 100644 --- a/tiered-storage/pom.xml +++ b/tiered-storage/pom.xml @@ -1,3 +1,4 @@ + - + 4.0.0 pom - org.apache.pulsar + io.streamnative pulsar - 2.10.5-SNAPSHOT + 2.10.5.3 .. - tiered-storage-parent Apache Pulsar :: Tiered Storage :: Parent - ${project.version} - jcloud file-system - diff --git a/tiered-storage/src/main/resources/findbugsExclude.xml b/tiered-storage/src/main/resources/findbugsExclude.xml index 5a9926c0e8116..75c291b6e9203 100644 --- a/tiered-storage/src/main/resources/findbugsExclude.xml +++ b/tiered-storage/src/main/resources/findbugsExclude.xml @@ -1,3 +1,4 @@ + - - - - + + + +