feat: Implementation Datax doriswriter plugin

extension/DataX/doriswriter/doc/doriswriter.md

@@ -0,0 +1,147 @@
+# DorisWriter 插件文档
+
+## 1 快速介绍
+DorisWriter支持将大批量数据写入Doris中。
+
+## 2 实现原理
+DorisWriter 通过Doris原生支持Stream load方式导入数据， DorisWriter会将`reader`读取的数据进行缓存在内存中，拼接成Json文本，然后批量导入至Doris。
+
+## 3 功能说明
+
+### 3.1 配置样例
+
+这里是一份从Stream读取数据后导入至Doris的配置文件。
+
+```
+{
+    "job": {
+        "setting": {
+            "speed": {
+                "channel": 1
+            },
+            "errorLimit": {
+                "record": 0,
+                "percentage": 0
+            }
+        },
+        "content": [
+            {
+                "reader": {
+                    "name": "streamreader",
+                    "parameter": {
+                        "column": [
+                            {
+                                "value": "皮蛋1",
+                                "type": "string"
+                            },
+                            {
+                                "value": "皮蛋2",
+                                "type": "string"
+                            },
+                            {
+                                "value": "111",
+                                "type": "long"
+                            },
+                            {
+                                "value": "222",
+                                "type": "long"
+                            }
+                        ],
+                        "sliceRecordCount": 100
+                    }
+                },
+                "writer": {
+                    "name": "doriswriter",
+                    "parameter": {
+                        "username": "dxx",
+                        "password": "123456",
+                        "database": "test",
+                        "table": "datax_test",
+                        "column": [
+                            "k1",
+                            "k2",
+                            "v1",
+                            "v2"
+                        ],
+                        "preSql": [],
+                        "postSql": [],
+                        "jdbcUrl": "jdbc:mysql://10.93.6.247:9030/",
+                        "beLoadUrl": [
+                            "10.93.6.167:8041"
+                        ],
+                        "loadProps": {
+                        }
+                    }
+                }
+            }
+        ]
+    }
+}
+```
+
+
+
+### 3.2 参数说明
+
+* **username**
+
+  - 描述：访问Doris数据库的用户名
+  - 必选：是
+  - 默认值：无
+
+* **password**
+
+  - 描述：访问Doris数据库的密码
+  - 必选：是
+  - 默认值：无
+
+* **database**
+
+  - 描述：访问Doris表的数据库名称。
+  - 必选：是
+  - 默认值：无
+
+* **table**
+
+  - 描述：访问Doris表的表名称。
+  - 必选：是
+  - 默认值：无
+
+* **beLoadUrl**
+
+  - 描述：Doris BE的地址用于Stream load，可以为多个BE地址，形如`BE_ip:Be_webserver_port`。
+  - 必选：是
+  - 默认值：无
+
+* **column**
+
+  - 描述：目的表**需要写入数据**的字段，字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。
+  - 必选：是
+  - 默认值：否
+
+* **preSql**
+
+  - 描述：写入数据到目的表前，会先执行这里的标准语句。
+  - 必选：否
+  - 默认值：无
+
+* **postSql**
+
+  - 描述：写入数据到目的表后，会执行这里的标准语句。
+  - 必选：否
+  - 默认值：无
+
+* **jdbcUrl**
+
+  - 描述：目的数据库的 JDBC 连接信息，用于执行`preSql`及`postSql`。
+  - 必选：否
+  - 默认值：无
+
+* **loadProps**
+
+  - 描述：StreamLoad 的请求参数，详情参照StreamLoad介绍页面。
+  - 必选：否
+  - 默认值：无
+
+
+

extension/DataX/doriswriter/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>datax-all</artifactId>
+        <groupId>com.alibaba.datax</groupId>
+        <version>0.0.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>doriswriter</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>datax-common</artifactId>
+            <version>${datax-project-version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>slf4j-log4j12</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.datax</groupId>
+            <artifactId>plugin-rdbms-util</artifactId>
+            <version>${datax-project-version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>${mysql.driver.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.3</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <!-- compiler plugin -->
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>${jdk-version}</source>
+                    <target>${jdk-version}</target>
+                    <encoding>${project-sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+
+            <!-- assembly plugin -->
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <descriptors>
+                        <descriptor>src/main/assembly/package.xml</descriptor>
+                    </descriptors>
+                    <finalName>datax</finalName>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>dwzip</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

extension/DataX/doriswriter/src/main/assembly/package.xml

@@ -0,0 +1,35 @@
+<assembly
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+    <id></id>
+    <formats>
+        <format>dir</format>
+    </formats>
+    <includeBaseDirectory>false</includeBaseDirectory>
+    <fileSets>
+        <fileSet>
+            <directory>src/main/resources</directory>
+            <includes>
+                <include>plugin.json</include>
+                <include>plugin_job_template.json</include>
+            </includes>
+            <outputDirectory>plugin/writer/doriswriter</outputDirectory>
+        </fileSet>
+        <fileSet>
+            <directory>target/</directory>
+            <includes>
+                <include>doriswriter-0.0.1-SNAPSHOT.jar</include>
+            </includes>
+            <outputDirectory>plugin/writer/doriswriter</outputDirectory>
+        </fileSet>
+    </fileSets>
+
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <outputDirectory>plugin/writer/doriswriter/libs</outputDirectory>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>

extension/DataX/doriswriter/src/main/java/com/alibaba/datax/plugin/writer/doriswriter/DorisFlushBatch.java

@@ -0,0 +1,47 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+  -->
+ */
+package com.alibaba.datax.plugin.writer.doriswriter;
+
+import java.util.List;
+
+public class DorisFlushBatch
+{
+	private String label;
+	private Long bytes;
+	private List<String> rows;
+
+	public DorisFlushBatch(final String label, final Long bytes, final List<String> rows) {
+		this.label = label;
+		this.bytes = bytes;
+		this.rows = rows;
+	}
+
+	public String getLabel() {
+		return this.label;
+	}
+
+	public Long getBytes() {
+		return this.bytes;
+	}
+
+	public List<String> getRows() {
+		return this.rows;
+	}
+}