Docker部署运行ELK

客官°小女子只卖身不卖艺 2023-06-13 03:29 29阅读 0赞

mkdir -p /home/docker/elasticsearch/data
    mkdir -p /home/docker/elasticsearch/logs
    mkdir -p /home/docker/logstash

**在 /home/docker/logstash目录下 vi logstash.conf**

input {
        kafka {
            bootstrap_servers => ["192.168.1.80:9092"] # 注意这里配置的kafka的broker地址不是zk的地址
            group_id => "applogs-msg" # 自定义groupid 
            topics => ["applogs"]  # kafka topic 名称 
            consumer_threads => 5 
            decorate_events => true
            codec => "json"
            tags => ["applogs1"]
            
          }
       # 这样可以配置多个topic,多个tags，如果不需要可以不加
       kafka {
            bootstrap_servers => ["192.168.1.80:9092"] # 注意这里配置的kafka的broker地址不是zk的地址
            group_id => "applogs-msg" # 自定义groupid
            topics => ["applogs1"]  # kafka topic 名称
            consumer_threads => 5
            decorate_events => true
            codec => "json"
            tags => ["applogs2"]
          }
    }
    
    output{
            elasticsearch{
                    hosts=>["192.168.1.80:9200"]  
                    index => "%{tags}-%{+YYYY.MM.dd}"
            }
            stdout{codec => rubydebug}
    }

**在/home/docker/logstash目录下， vi logstash.yml**

path.config: /etc/logstash/conf.d

**在/home/docker 目录下，vi docker-elk.yml**

version: '3.4'
    services:
     elasticsearch:
      image: elasticsearch:7.4.2
      container_name: elasticsearch-single
      restart: always
      ulimits:
        nproc: 65535
        nofile:
          soft: 40000
          hard: 65535 
        memlock:
          soft: -1
          hard: -1
      ports:
        - "9200:9200"
        - "9300:9300"
      environment:
        - TZ=Asia/Shanghai
        - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
        - bootstrap.memory_lock=true
        - http.host=0.0.0.0
        #- cluster.initial_master_nodes=["node-1"]
        #- bootstrap.memory_lock=false
        - bootstrap.system_call_filter=false
        - discovery.type=single-node #单机运行
      volumes:
        #- /home/docker/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml 
        - /home/docker/elasticsearch/data:/usr/share/elasticsearch/data
        - /home/docker/elasticsearch/logs:/usr/share/elasticsearch/logs 
        #- /home/docker/elasticsearch/plugins:/usr/share/elasticsearch/plugins 
    
     logstash:
      image: logstash:7.4.2
      container_name: logstash-single 
      restart: always
      #network_mode: "bridge"
      #command: logstash -f /config-dir
      ports:
        - "5044:5044"
      volumes:
        - /home/docker/logstash/logstash.conf:/opt/logstash/config/logstash.conf    
        - /home/docker/logstash/logstash.yml:/opt/logstash/config/logstash.yml   
        - /home/docker/logstash/logstash.conf:/etc/logstash/conf.d/logstash.conf
        - /home/docker/logstash/logstash.yml:/etc/logstash/logstash.yml
      environment:
          # 设置时区
        TZ: Asia/Shanghai
      external_links:
        - elasticsearch:elasticsearch-single
    
     kibana:
      image: kibana:7.4.2
      container_name: kibana-single 
      restart: always
      #network_mode: "bridge"
      ports:
        - "5601:5601"
      external_links:
        - elasticsearch:elasticsearch-single

**最后docker-compose -f docker-elk.yml up -d**

**部署遇到的问题**

* soft noproc 11000
    * hard noproc 11000
    * soft nofile 4100
    * hard nofile 4100 
    * - nofile65536 
    * - memlock unlimited
    
    max virtual memory areas vm.max_map_count [65530]is too low, increase to at least [262144]
    这个错误，如果是linux直接部署es的话
    vi /etc/sysctl.conf
    添加
    vm.max_map_count=655360
    
    max file descriptors [65535] for elasticsearchprocess is too low, increase to at least [65536]
    这个错误，如果是linux直接部署es的话
    vi /etc/security/limits.conf
    * soft noproc 11000
    * hard noproc 11000
    * soft nofile 4100
    * hard nofile 4100 
    * - nofile65536 
    * - memlock unlimited
    
    执行 sysctl -p 立即生效
    
    docker部署的话，以上这里两个错误需要修改配置
     ulimits:
        nproc: 65535
        nofile:
          soft: 40000
          hard: 65535 
        memlock:
          soft: -1
          hard: -1
    docker-compose.yml里加上这个
    
    the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configure
    这个错误是es配置的问题
    docker-compose.yml加上这个，或者 在es的配置文件里加
    environment:
      - bootstrap.system_call_filter: false
      - cluster.initial_master_nodes: ["node-1"]