From 3fed8b621dd5899889cdbe759feffb380b0132a6 Mon Sep 17 00:00:00 2001 From: streamnativebot Date: Wed, 6 Mar 2024 06:21:34 +0000 Subject: [PATCH] "Created by snbot" --- docs/lakehouse-sink.md | 18 ++++---- docs/lakehouse-source.md | 10 ++--- pom.xml | 90 ++++++++++++---------------------------- 3 files changed, 41 insertions(+), 77 deletions(-) diff --git a/docs/lakehouse-sink.md b/docs/lakehouse-sink.md index ccba18c..fd9c92c 100644 --- a/docs/lakehouse-sink.md +++ b/docs/lakehouse-sink.md @@ -42,7 +42,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-2.10.5.15.nar ``` # How to configure @@ -133,7 +133,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-2.10.5.15.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -156,7 +156,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-2.10.5.15.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -185,7 +185,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -213,7 +213,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -246,7 +246,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -268,7 +268,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -339,7 +339,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.10.5.15.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.15.nar ``` 2. Start Pulsar in standalone mode. @@ -372,7 +372,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.10.5.15.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.15.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/docs/lakehouse-source.md b/docs/lakehouse-source.md index 8195c07..a561d45 100644 --- a/docs/lakehouse-source.md +++ b/docs/lakehouse-source.md @@ -42,7 +42,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-2.10.5.15.nar ``` # How to configure @@ -92,7 +92,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -117,7 +117,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.15.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -178,7 +178,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.10.5.15.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.15.nar ``` 2. Start Pulsar in standalone mode. @@ -209,7 +209,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.10.5.15.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.15.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/pom.xml b/pom.xml index 4e1f451..9b03624 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + org.apache apache 18 - 4.0.0 - org.apache.pulsar.ecosystem pulsar-io-lakehouse - 2.10.5-SNAPSHOT + 2.10.5.15 Pulsar Ecosystem :: IO Connector :: Lake House It is a lakehouse connector streaming convert data between lakehouse and Apache Pulsar. - 1.8 1.8 @@ -45,11 +40,10 @@ true 3.1.8 0 - 2.13.2.1 1.18.22 - 2.10.0.7 + 2.10.5.15 2.17.2 1.7.25 3.2.4 @@ -63,12 +57,10 @@ hadoop3-2.2.1 2.12.0 1.1.8.4 - 7.3.0 4.0.3 3.12.4 - 3.0 3.1.2 @@ -77,9 +69,8 @@ 1.2.0 8.37 3.1.8 - + - Apache License, Version 2.0 @@ -87,7 +78,6 @@ repo - @@ -181,8 +171,6 @@ log4j-slf4j-impl ${log4j2.version} - - org.apache.iceberg @@ -214,20 +202,17 @@ iceberg-hive-metastore ${iceberg.version} - org.apache.parquet parquet-avro ${parquet.avro.version} - io.delta delta-standalone_2.12 ${delta.version} - org.apache.parquet parquet-column @@ -238,7 +223,6 @@ parquet-hadoop ${parquet.version} - org.slf4j slf4j-api @@ -249,13 +233,11 @@ slf4j-simple ${slf4j.version} - org.xerial.snappy snappy-java ${snappy.java.version} - org.apache.hudi @@ -269,7 +251,6 @@ - org.apache.hadoop @@ -287,14 +268,12 @@ aws-java-sdk-bundle ${aws.sdk.version} - com.google.cloud.bigdataoss gcs-connector ${gcs.version} shaded - org.apache.hadoop hadoop-azure @@ -310,10 +289,7 @@ - - - org.testng @@ -339,13 +315,10 @@ ${mockito.version} test - - - org.projectlombok @@ -357,13 +330,11 @@ spotbugs-annotations provided - io.streamnative pulsar-io-core - com.fasterxml.jackson.core jackson-databind @@ -377,37 +348,30 @@ org.apache.hadoop hadoop-client - org.apache.iceberg iceberg-api - org.apache.iceberg iceberg-core - org.apache.iceberg iceberg-parquet - org.apache.iceberg iceberg-common - org.apache.iceberg iceberg-data - org.apache.iceberg iceberg-hive-metastore - io.delta delta-standalone_2.12 @@ -416,7 +380,6 @@ org.apache.parquet parquet-avro - org.apache.parquet parquet-column @@ -425,7 +388,6 @@ org.apache.parquet parquet-hadoop - org.slf4j slf4j-api @@ -434,12 +396,10 @@ org.slf4j slf4j-simple - org.xerial.snappy snappy-java - org.apache.logging.log4j log4j-slf4j-impl @@ -460,12 +420,10 @@ org.apache.logging.log4j log4j-slf4j-impl - io.netty netty-common - org.apache.hudi hudi-java-client @@ -484,7 +442,6 @@ - org.apache.curator curator-framework @@ -496,7 +453,6 @@ - org.apache.hudi hudi-client-common @@ -510,7 +466,6 @@ - org.testng @@ -528,7 +483,6 @@ test - cloud @@ -552,13 +506,11 @@ com.amazonaws aws-java-sdk-bundle - com.google.cloud.bigdataoss gcs-connector shaded - org.apache.hadoop hadoop-azure @@ -566,7 +518,6 @@ - ${project.artifactId}-${project.version}${cloudPkgSuffix} @@ -583,7 +534,7 @@ -Xlint:unchecked -Xpkginfo:always - + @@ -695,38 +646,32 @@ - - + org.apache.maven.plugins maven-compiler-plugin - org.apache.maven.plugins maven-surefire-plugin - org.apache.nifi nifi-nar-maven-plugin - com.mycila license-maven-plugin - org.apache.maven.plugins maven-checkstyle-plugin - com.github.spotbugs @@ -734,5 +679,24 @@ - + + + ossrh + https://s01.oss.sonatype.org/service/local/repositories/iostreamnative-2888/content + + + nexus-snapshot + https://s01.oss.sonatype.org/content/repositories/snapshots + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + +