Dockerfile 构建 Flume 镜像

布满荆棘的人生 2023-01-02 04:30 406阅读 0赞

> flume version: 1.9.0

# 1. 依赖的服务/组件  #

> \- java8 
> 
> \- flume 1.9.0
> 
> \- supervisor-4.2.1
> 
>     - setuptools-44.0.0           \# 这里是基于centos7的镜像，内置python版本为2.7.6，所以需要一个支持py2的setuptools版本
> 
>     - meld3-2.0.1

# 2. 下载离线安装包 #

<table> 
 <tbody> 
  <tr> 
   <td style="width:222px;">jdk8</td> 
   <td><a href="https://repo.huaweicloud.com/java/jdk/8u202-b08/jdk-8u202-linux-x64.tar.gz" rel="nofollow">https://repo.huaweicloud.com/java/jdk/8u202-b08/jdk-8u202-linux-x64.tar.gz</a></td> 
  </tr> 
  <tr> 
   <td style="width:222px;">flume 1.9.0</td> 
   <td><a href="https://mirrors.tuna.tsinghua.edu.cn/apache/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz" rel="nofollow">https://mirrors.tuna.tsinghua.edu.cn/apache/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz</a></td> 
  </tr> 
  <tr> 
   <td style="width:222px;">supervisor-4.2.1</td> 
   <td><a href="https://files.pythonhosted.org/packages/44/60/698e54b4a4a9b956b2d709b4b7b676119c833d811d53ee2500f1b5e96dc3/supervisor-3.3.4.tar.gz" rel="nofollow">https://files.pythonhosted.org/packages/44/60/698e54b4a4a9b956b2d709b4b7b676119c833d811d53ee2500f1b5e96dc3/supervisor-3.3.4.tar.gz</a></td> 
  </tr> 
  <tr> 
   <td style="width:222px;">setuptools-44.0.0&nbsp;</td> 
   <td><a href="https://mirrors.aliyun.com/pypi/packages/b0/f3/44da7482ac6da3f36f68e253cb04de37365b3dba9036a3c70773b778b485/setuptools-44.0.0.zip#sha256=e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008" rel="nofollow">https://mirrors.aliyun.com/pypi/packages/b0/f3/44da7482ac6da3f36f68e253cb04de37365b3dba9036a3c70773b778b485/setuptools-44.0.0.zip#sha256=e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008</a></td> 
  </tr> 
  <tr> 
   <td style="width:222px;">meld3-2.0.1</td> 
   <td><a href="https://mirrors.aliyun.com/pypi/packages/53/af/5b8b67d04a36980de03505446d35db39c7b2a01b9bac1cb673434769ddb8/meld3-2.0.1.tar.gz#sha256=3ea266994f1aa83507679a67b493b852c232a7905e29440a6b868558cad5e775" rel="nofollow">https://mirrors.aliyun.com/pypi/packages/53/af/5b8b67d04a36980de03505446d35db39c7b2a01b9bac1cb673434769ddb8/meld3-2.0.1.tar.gz#sha256=3ea266994f1aa83507679a67b493b852c232a7905e29440a6b868558cad5e775</a></td> 
  </tr> 
 </tbody> 
</table>

# 3.  Dockerfile #

## 3.1 将下载的离线安装包放到统一的目录下，目录结构为： ##

mit@mit:~/my_project/big_data/data_collect/flume/Dockerfile$ tree
    .
    ├── apache-flume-1.9.0-bin.tar.gz
    ├── Dockerfile
    ├── jdk-8u202-linux-x64.tar.gz
    ├── meld3-2.0.1.tar.gz
    ├── setuptools-44.0.0.tar.gz
    ├── start.sh
    ├── supervisor-4.2.1.tar.gz
    └── supervisord.conf

##  3.2 supervisord.conf ##

supervisord.conf 是supervisord的配置文件，只是将最后的 \[include\] 配置项取消注释，指定到配置存放目录

[include]
    files = /etc/supervisor/config.d/*.conf

## 3.3 Dockerfile ##

FROM centos:centos7
    LABEL MAINTAINER=mit description="FLume-ng数据采集agent" FlumeVersion=1.9.0
    # 安装 java 环境
    ADD jdk-8u202-linux-x64.tar.gz /usr/local/java
    ENV JAVA_HOME /usr/local/java/jdk1.8.0_202
    ENV CLASSPATH $JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    ENV PATH $JAVA_HOME/bin:$PATH
    # 安装 flume
    ADD apache-flume-1.9.0-bin.tar.gz /usr/local/
    RUN mv /usr/local/apache-flume-1.9.0-bin /usr/local/flume
    ENV FLUME_HOME /usr/local/flume
    ENV PATH $FLUME_HOME/bin:$PATH
    # 修改默认的flume-ng jvm heap大小
    RUN sed -i '225 s/Xmx20m/Xmx2048m/' /usr/local/flume/bin/flume-ng
    # 安装 supervisor
    # - setuptools
    ADD setuptools-44.0.0.tar.gz /opt
    WORKDIR /opt/setuptools-44.0.0/
    RUN python setup.py install
    # - meld3
    ADD meld3-2.0.1.tar.gz /opt
    WORKDIR /opt/meld3-2.0.1/
    RUN python setup.py install
    # - supervisor
    ADD supervisor-4.2.1.tar.gz /opt
    WORKDIR /opt/supervisor-4.2.1/
    RUN python setup.py install
    RUN mkdir -p /etc/supervisor && mkdir -p /etc/supervisor/config.d && rm -rf /opt/setuptools-44.0.0 /opt/meld3-2.0.1 /opt/supervisor-4.2.1
    COPY supervisord.conf /etc/supervisor/
    ENTRYPOINT ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"]
    # 设定工作目录
    WORKDIR /usr/local/flume

# 4. 构建Flume镜像 #

docker build -f Dockerfile -t flume:1.9.0 .

# 5. 基于构建的镜像创建容器 #

## 5.1 验证镜像 ##

# 创建新容器
    docker run -d -it --name flume flume:1.9.0

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDIzODYyNQ_size_16_color_FFFFFF_t_70][]

## 5.2 Flume镜像的使用 ##

上面构建的镜像只是将相应的服务打进了镜像里，使用时应挂载相应flume-ng配置和supervisor应用配置，以下为我使用docker-compose启动flume服务的相应docker-compose.yaml部分配置

flume-01:
        image: flume:1.9.0
        container_name: flume-01
        hostname: flume-01
        restart: always
        ports:
          - "6000:6000"   # flume-ng source监听的端口
        volumes:
          - /etc/localtime:/etc/localtime:ro
          - /home/mit/my_project/big_data/data_collect/flume/supervisord.d/flume.conf:/etc/supervisord.d/flume.conf              # flume守护
          - /home/mit/my_project/big_data/data_collect/flume/conf/http_kafka_channel.conf:/usr/local/flume/conf/http_kafka_channel.conf          # flume-ng agent启动的配置文件
          - /home/mit/my_project/big_data/data_collect/flume/conf/log4j.properties:/usr/local/flume/conf/log4j.properties
          - /home/mit/my_project/big_data/data_collect/flume/data:/usr/local/flume/data                                          # FileChannel data dir
          - /home/mit/my_project/big_data/data_collect/flume/checkpoint:/usr/local/flume/checkpoint                              # FIleChannel checkpoint dir
          - /home/mit/my_project/big_data/data_collect/flume/logs:/usr/local/flume/logs                                          # flume-ng agent日志
          - /home/mit/my_project/big_data/data_collect/flume/lib/fastjson-1.2.59.jar:/usr/local/flume/lib/fastjson-1.2.59.jar    # flume-ng 自定义拦截器依赖的jar包需要全部列出
          - /home/mit/my_project/big_data/data_collect/flume/lib/TQDataInterceptor-0.1.jar:/usr/local/flume/lib/TQDataInterceptor-0.1.jar
        environment:
          JAVA_HOME: "/usr/java/jdk1.8.0_202-amd64/"
        depends_on:
          - kafka-01
          - kafka-02
          - kafka-03

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDIzODYyNQ_size_16_color_FFFFFF_t_70]: /images/20221120/76264c9578be4f438b5978849836cfc9.png