jiafeng.zhang
@ -20,7 +20,7 @@ under the License.
|
||||
<template>
|
||||
<div class="article-wrap">
|
||||
<Navbar />
|
||||
<div class="content-wrapper">
|
||||
<div class="content-wrapper1">
|
||||
<div class="article-header">
|
||||
<h1>{{frontmatter.title}}</h1>
|
||||
<div class="article-sub-header">
|
||||
@ -56,10 +56,10 @@ export default {
|
||||
};
|
||||
</script>
|
||||
<style lang="stylus">
|
||||
.content-wrapper
|
||||
.content-wrapper1
|
||||
padding: 60px 15px 80px
|
||||
min-height: calc(100vh - 140px)
|
||||
max-width: 740px
|
||||
max-width: 80%
|
||||
margin: 0 auto
|
||||
.article-sub-header
|
||||
.article-header-item
|
||||
|
||||
@ -64,7 +64,7 @@ export default {
|
||||
},
|
||||
computed: {
|
||||
articleList () {
|
||||
return this.$site.pages.filter(item => item.frontmatter.metaTitle === 'article' && item.frontmatter.language === this.$lang)
|
||||
return this.$site.pages.filter(item => item.frontmatter.isArticle && item.frontmatter.language === this.$lang)
|
||||
},
|
||||
currentArticleList () {
|
||||
const start = (this.pageNumber - 1) * this.pageSize
|
||||
@ -89,7 +89,7 @@ export default {
|
||||
.content-wrapper
|
||||
padding: 100px 15px 80px
|
||||
min-height: calc(100vh - 180px)
|
||||
max-width: 740px
|
||||
max-width: 80%
|
||||
margin: 0 auto
|
||||
.article-item
|
||||
padding-bottom: 20px
|
||||
|
||||
@ -3,9 +3,10 @@
|
||||
"title": "How to use Apache doris Datax DorisWriter extension",
|
||||
"description": "In order to better expand the Apache doris ecosystem and provide more convenient data import for doris users, the community development and extension supports Datax DorisWriter, making it more convenient for Datax to access data.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "article",
|
||||
"language": "zh_cn",
|
||||
"metaTitle": "How to use Apache doris Datax DorisWriter extension",
|
||||
"language": "en",
|
||||
"author": "张家锋",
|
||||
"isArticle": true,
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
}
|
||||
@ -29,7 +30,6 @@ KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# How to use Apache doris Datax DorisWriter extension
|
||||
|
||||
DataX is an open source version of Alibaba Cloud DataWorks data integration, an offline data synchronization tool/platform widely used in Alibaba Group. DataX implements efficient data synchronization functions between various heterogeneous data sources including MySQL, Oracle, SqlServer, Postgre, HDFS, Hive, ADS, HBase, TableStore (OTS), MaxCompute (ODPS), Hologres, DRDS, etc.
|
||||
|
||||
@ -96,7 +96,7 @@ Here I am performing the compilation of Datax into a tar package, which is not t
|
||||
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/image-20210903132250723.png" alt="image-20210903132250723" style="zoom:50%;" />
|
||||
|
||||
|
||||
|
||||
@ -250,7 +250,7 @@ python bin/datax.py doris.json
|
||||
|
||||
Then you can see the execution result:
|
||||
|
||||
|
||||
<img src="/images/image-20210903134043421.png" alt="image-20210903134043421" style="zoom:30%;" />
|
||||
|
||||
Then go to the Doris database to check your table, the data has been imported, and the task execution is over
|
||||
|
||||
|
||||
@ -3,9 +3,10 @@
|
||||
"title": "Apache Doris FE load balance",
|
||||
"description": "In order to better expand the Apache doris ecosystem and provide more convenient data import for doris users, the community development and extension supports Datax DorisWriter, making it more convenient for Datax to access data.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "article",
|
||||
"language": "zh_cn",
|
||||
"metaTitle": "Apache Doris FE load balance",
|
||||
"language": "en",
|
||||
"author": "张家锋",
|
||||
"isArticle": true,
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
}
|
||||
@ -30,8 +31,6 @@ specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# FE load balance
|
||||
|
||||
When deploying multiple FE nodes, users can deploy a load balancing layer on top of multiple FEs to achieve high availability of Doris.
|
||||
|
||||
## 1. Code method
|
||||
|
||||
@ -3,10 +3,11 @@
|
||||
"title": "Use Flink CDC to realize real-time MySQL data into Apache Doris",
|
||||
"description": "This article uses examples to demonstrate how to use Flink CDC and Doris' Flink Connector to monitor data from the Mysql database and store it in the corresponding table of the Doris data warehouse in real time.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "article",
|
||||
"language": "zh_cn",
|
||||
"metaTitle": "Use Flink CDC to realize real-time MySQL data into Apache Doris",
|
||||
"language": "en",
|
||||
"author": "张家锋",
|
||||
"layout": "Article",
|
||||
"isArticle": true,
|
||||
"sidebar": false
|
||||
}
|
||||
---
|
||||
|
||||
@ -37,15 +37,20 @@ To write a blog file, you must first include Front Matter information in the hea
|
||||
| description | - | Blog description|
|
||||
| date | - | Blog date |
|
||||
| author | - | Blog author |
|
||||
| metaTitle | article | Mark blog file |
|
||||
| metaTitle | - | The title displayed by the browser when browsing the article |
|
||||
| language | en/zn-CN | language |
|
||||
| layout | Article | Layout of the components |
|
||||
| sidebar | false | Hide the sidebar |
|
||||
| isArticle | true | Whether it is an article, do not modify by default |
|
||||
|
||||
The title, description and date fields are filled in by the blog writer, while the other fields are fixed values。
|
||||
>**Attention**
|
||||
>
|
||||
>The title, description, date, author, and metaTitle field values are filled in by the blog writer, and the other fields are fixed values.
|
||||
>
|
||||
>language: en, zh-CN, mainly don’t make mistakes
|
||||
|
||||
File header example：
|
||||
```
|
||||
```json
|
||||
---
|
||||
{
|
||||
"title": "This is title",
|
||||
@ -56,6 +61,7 @@ File header example：
|
||||
"language": "zh-CN",
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
"isArticle":true
|
||||
}
|
||||
---
|
||||
```
|
||||
|
||||
@ -3,8 +3,9 @@
|
||||
"title": "Apache Doris Datax DorisWriter扩展使用方法",
|
||||
"description": "为了更好的扩展Apache doris生态，为doris用户提供更方便的数据导入，社区开发扩展支持了Datax DorisWriter，使大家更方便Datax进行数据接入.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "article",
|
||||
"language": "zh_cn",
|
||||
"metaTitle": "Apache Doris Datax DorisWriter扩展使用方法",
|
||||
"isArticle": true,
|
||||
"language": "zh-CN",
|
||||
"author": "张家锋",
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
@ -29,7 +30,6 @@ KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# Datax DorisWriter扩展使用方法
|
||||
|
||||
DataX 是阿里云 DataWorks数据集成 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS 等各种异构数据源之间高效的数据同步功能
|
||||
|
||||
@ -98,7 +98,7 @@ oceanbasev10writer
|
||||
mvn -U clean package assembly:assembly -Dmaven.test.skip=true
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/image-20210903132250723.png" alt="image-20211105134818474" style="zoom:50%;" />
|
||||
|
||||
|
||||
|
||||
@ -254,7 +254,7 @@ python bin/datax.py doris.json
|
||||
|
||||
然后就可以看到执行结果：
|
||||
|
||||
|
||||
<img src="/images/image-20210903134043421.png" alt="image-20210903134043421" style="zoom:30%;" />
|
||||
|
||||
再去 Doris 数据库中查看你的表，数据就已经导入进去了，任务执行结束
|
||||
|
||||
|
||||
@ -1,7 +1,14 @@
|
||||
---
|
||||
{
|
||||
"title": "FE负载均衡",
|
||||
"language": "zh-CN"
|
||||
"title": "Apache Doris FE负载均衡",
|
||||
"description": "当部署多个 FE 节点时，用户可以在多个 FE 之上部署负载均衡层来实现 Doris 的高可用.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "Apache Doris FE负载均衡",
|
||||
"isArticle": true,
|
||||
"language": "zh-CN",
|
||||
"author": "张家锋",
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
}
|
||||
---
|
||||
|
||||
@ -24,8 +31,6 @@ specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# FE负载均衡
|
||||
|
||||
当部署多个 FE 节点时，用户可以在多个 FE 之上部署负载均衡层来实现 Doris 的高可用。
|
||||
|
||||
## 1. 代码方式
|
||||
|
||||
@ -3,8 +3,9 @@
|
||||
"title": "使用 Flink CDC 实现 MySQL 数据实时入 Apache Doris",
|
||||
"description": "本文通过实例来演示怎么通过Flink CDC 结合Doris的Flink Connector实现从Mysql数据库中监听数据并实时入库到Doris数仓对应的表中.",
|
||||
"date": "2021-11-11",
|
||||
"metaTitle": "article",
|
||||
"language": "zh_cn",
|
||||
"metaTitle": "使用 Flink CDC 实现 MySQL 数据实时入 Apache Doris",
|
||||
"isArticle": true,
|
||||
"language": "zh-CN",
|
||||
"author": "张家锋",
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
@ -28,7 +29,6 @@ KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
# 使用 Flink CDC 实现 MySQL 数据实时入 Apache Doris
|
||||
|
||||
本文通过实例来演示怎么通过Flink CDC 结合Doris的Flink Connector实现从Mysql数据库中监听数据并实时入库到Doris数仓对应的表中。
|
||||
|
||||
@ -176,7 +176,7 @@ https://repo1.maven.org/maven2/com/ververica/flink-connector-mysql-cdc/2.0.2/fli
|
||||
# wget https://github.com/hf200012/hf200012.github.io/raw/main/lib/doris-flink-1.0-SNAPSHOT.jar -P ./lib/
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211026095513892.png" alt="image-20210903134043421" style="zoom:50%;" />
|
||||
|
||||
#### 4.2.2 启动Flink
|
||||
|
||||
@ -191,7 +191,7 @@ Starting taskexecutor daemon on host doris01.
|
||||
|
||||
我们通过web访问（默认端口是8081）启动起来Flink 集群，可以看到集群正常启动
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211025162831632.png" alt="image-20211025162831632" style="zoom:30%;" />
|
||||
|
||||
### 4.3 安装Apache Doris
|
||||
|
||||
@ -257,7 +257,7 @@ CREATE TABLE `doris_test` (
|
||||
> set execution.result-mode=tableau;
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211025165547903.png" alt="image-20211025165547903" style="zoom:30%;" />
|
||||
|
||||
#### 4.5.1 创建 Flink CDC Mysql 映射表
|
||||
|
||||
@ -283,7 +283,7 @@ CREATE TABLE test_flink_cdc (
|
||||
select * from test_flink_cdc;
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211026100505972.png" alt="image-20211025165547903" style="zoom:30%;" />
|
||||
|
||||
#### 4.5.2 创建Flink Doris Table 映射表
|
||||
|
||||
@ -319,11 +319,11 @@ select * from doris_test_sink;
|
||||
INSERT INTO doris_test_sink select id,name from test_flink_cdc
|
||||
```
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211026101004547.png" alt="image-20211025165547903" style="zoom:50%;" />
|
||||
|
||||
提交成功之后我们在Flink的Web界面可以看到相关的Job任务信息
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211026100943474.png" alt="image-20211025165547903" style="zoom:30%;" />
|
||||
|
||||
#### 4.5.3 向Mysql表中插入数据
|
||||
|
||||
@ -352,7 +352,7 @@ INSERT INTO test_cdc VALUES (91233, 'zhangfeng_9');
|
||||
|
||||
重新启动Insert into任务
|
||||
|
||||
|
||||
<img src="/images/cdc/image-20211025182341086.png" alt="image-20211025182341086" style="zoom:50%;" />
|
||||
|
||||
修改Mysql表里的数据
|
||||
|
||||
|
||||
@ -37,12 +37,17 @@ under the License.
|
||||
| description | - | Blog描述|
|
||||
| date | - | Blog发布时间 |
|
||||
| author | - | Blog作者 |
|
||||
| metaTitle | article | 标记Blog文件 |
|
||||
| language | en/zn-CN | 语言 |
|
||||
| metaTitle | - | 浏览文章时候浏览器显示的标题 |
|
||||
| language | en/zh-CN | 语言 |
|
||||
| layout | Article | 布局组件 |
|
||||
| sidebar | false | 隐藏侧边栏 |
|
||||
| isArticle | true | 是否是文章，默认不要修改 |
|
||||
|
||||
其中title、description和date字段值由Blog编写者填写，其他字段为固定值。
|
||||
>**注意**
|
||||
>
|
||||
>其中title、description、date、author，metaTitle字段值由Blog编写者填写，其他字段为固定值。
|
||||
>
|
||||
>language：en，zh-CN，主要不要书写错误
|
||||
|
||||
文件头示例：
|
||||
```
|
||||
@ -52,10 +57,11 @@ under the License.
|
||||
"description": "This is description",
|
||||
"date": "2021-11-03",
|
||||
"author": "Alex",
|
||||
"metaTitle": "article",
|
||||
"metaTitle": "This is title",
|
||||
"language": "zh-CN",
|
||||
"layout": "Article",
|
||||
"sidebar": false
|
||||
"isArticle":true
|
||||
}
|
||||
---
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user