a6b51ec19a
support read avro file by hdfs() or s3() .
```sql
select * from s3(
"uri" = "http://127.0.0.1:9312/test2/person.avro",
"ACCESS_KEY" = "ak",
"SECRET_KEY" = "sk",
"FORMAT" = "avro");
+--------+--------------+-------------+-----------------+
| name | boolean_type | double_type | long_type |
+--------+--------------+-------------+-----------------+
| Alyssa | 1 | 10.0012 | 100000000221133 |
| Ben | 0 | 5555.999 | 4009990000 |
| lisi | 0 | 5992225.999 | 9099933330 |
+--------+--------------+-------------+-----------------+
select * from hdfs(
"uri" = "hdfs://127.0.0.1:9000/input/person2.avro",
"fs.defaultFS" = "hdfs://127.0.0.1:9000",
"hadoop.username" = "doris",
"format" = "avro");
+--------+--------------+-------------+-----------+
| name | boolean_type | double_type | long_type |
+--------+--------------+-------------+-----------+
| Alyssa | 1 | 8888.99999 | 89898989 |
+--------+--------------+-------------+-----------+
```
current avro reader only support common data type, the complex data types will be supported later.
105 lines
3.7 KiB
XML
105 lines
3.7 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one
|
|
or more contributor license agreements. See the NOTICE file
|
|
distributed with this work for additional information
|
|
regarding copyright ownership. The ASF licenses this file
|
|
to you under the Apache License, Version 2.0 (the
|
|
"License"); you may not use this file except in compliance
|
|
with the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing,
|
|
software distributed under the License is distributed on an
|
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
KIND, either express or implied. See the License for the
|
|
specific language governing permissions and limitations
|
|
under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<parent>
|
|
<artifactId>be-java-extensions</artifactId>
|
|
<groupId>org.apache.doris</groupId>
|
|
<version>${revision}</version>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<artifactId>avro-scanner</artifactId>
|
|
|
|
<properties>
|
|
<maven.compiler.source>8</maven.compiler.source>
|
|
<maven.compiler.target>8</maven.compiler.target>
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
</properties>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.doris</groupId>
|
|
<artifactId>java-common</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-tools</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.amazonaws</groupId>
|
|
<artifactId>aws-java-sdk-s3</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.doris</groupId>
|
|
<artifactId>hive-catalog-shade</artifactId>
|
|
</dependency>
|
|
</dependencies>
|
|
|
|
|
|
<build>
|
|
<finalName>avro-scanner</finalName>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-assembly-plugin</artifactId>
|
|
<configuration>
|
|
<descriptors>
|
|
<descriptor>src/main/resources/package.xml</descriptor>
|
|
</descriptors>
|
|
<archive>
|
|
<manifest>
|
|
<mainClass></mainClass>
|
|
</manifest>
|
|
</archive>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>make-assembly</id>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>single</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</project> |