[root@master pyflink]# cat test.txt
aaaaa 111111
bbbbb 222222
ccccc 333333
ddddd 444444
eeeee 555555
[root@master pyflink]# cat test.py
# -*- coding: utf-8 -*-
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from abc import ABC, abstractmethod
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from pyflink.datastream.state import MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer
from pyflink.common.typeinfo import Types, TypeInformation
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
import json
import re
from datetime import datetime
from elasticsearch import Elasticsearch
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction
import re
import redis
# 创建 StreamExecutionEnvironment 对象
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
# 读取文件,创建 DataStream 对象
data_stream = env.read_text_file('/root/pyflink/test.txt')
class MyMapFunction(FlatMapFunction):
def open(self, runtime_context: RuntimeContext):
pass
def close(self):
pass
def flat_map(self,line):
r_value=str(int(line.split(' ')[1]) + 1)
dict1={}
dict1['r_value']=r_value
yield dict1
env.add_jars("file:///root/lib/flink-sql-connector-elasticsearch7-3.0.1-1.16.jar")
date_str = datetime.now().strftime("%Y-%m-%d")
es7_sink = Elasticsearch7SinkBuilder() \
.set_bulk_flush_max_actions(1) \
.set_emitter(ElasticsearchEmitter.static_index('flink-test2023-06-07')) \
.set_hosts(['127.0.0.1:9200']) \
.build()
new_stream = data_stream.flat_map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING())).sink_to(es7_sink)
# 输出到控制台文章来源:https://www.toymoban.com/news/detail-488183.html
# 执行任务
env.execute('Add "bus_seq" to each line')文章来源地址https://www.toymoban.com/news/detail-488183.html
到了这里,关于python flat_map字典写入es的文章就介绍完了。如果您还想了解更多内容,请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章,希望大家以后多多支持TOY模板网!