Flink 在 idea上提交任务到远程服务器

忘是亡心i 2023-02-22 12:14 76阅读 0赞

###### Flink自身提供了远程提交任务的环境,源码如下: ######

请查看StreamExecutionEnvironment 类中 createRemoteEnvironment 方法

def createRemoteEnvironment(
        host: String,
        port: Int,
        parallelism: Int,
        jarFiles: String*): StreamExecutionEnvironment = {
    
      val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
      javaEnv.setParallelism(parallelism)
      new StreamExecutionEnvironment(javaEnv)
    }

###### 远程提交示例代码如下: ######

package com.flink.remotesubmit
    
    import org.apache.flink.streaming.api.scala._
    
    object RemoteSubmitApp extends App {
    
      val host: String = "node02"
      val port: Int = 8081
      val jarFiles = "E:\\CDHProjectDemo\\flink-demo\\target\\flink-demo-0.0.1-SNAPSHOT.jar"
    
      val env = StreamExecutionEnvironment.createRemoteEnvironment(host, port, jarFiles)
    
      val socketHost: String = "node01"
      val socketPort: Int = 7777
      val socketDs: DataStream[String] = env.socketTextStream(socketHost, socketPort)
    
      socketDs.flatMap(_.split(" "))
        .map((_, 1))
        .keyBy(0)
        .sum(1)
        .print()
    
      env.execute("Remote Submit Job")
    }

**注意**:

1.  需要保持代码和jar一致性，意思就是修改代码之后需重新执行 `mvn clean package`
2.  需在项目的 `src/main/resource` 目录中添加相关配置文件(`core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml`等)

###### 运行情况 ######

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA_size_16_color_FFFFFF_t_70][]

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA_size_16_color_FFFFFF_t_70 1][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200703132514918.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA==,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/20200703132548733.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzIxNTI1MA==,size_16,color_FFFFFF,t_70