CentOS6 Install kafka

落日映苍穹つ 2022-04-04 04:47 96阅读 0赞

**一.zookeeper集群安装**

原文链接：[http://www.cnblogs.com/caoguo/p/5958608.html][http_www.cnblogs.com_caoguo_p_5958608.html]

\# 环境

192.168.190.152
    192.168.190.153
    192.168.190.154

Install jdk

![复制代码][copycode.gif]

# tar zxvf jdk-8u65-linux-x64.tar.gz
    # mv jdk1.8.0_65 /usr/local/
    # vi /etc/profile.d/jdk.sh 
    JAVA_HOME=/usr/local/jdk1.8.0_65
    export JAVA_HOME
    PATH=$JAVA_HOME/bin:$PATH
    export PATH
    
    # source /etc/profile

![复制代码][copycode.gif]

Install zookeeper

![复制代码][copycode.gif]

# wget http://www-eu.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz
    
    # tar zxvf zookeeper-3.4.9.tar.gz
    # mv zookeeper-3.4.9 /usr/local/
    # useradd zookeeper
    # chown -R zookeeper.zookeeper /usr/local/zookeeper-3.4.9
    
    # cd /usr/local/zookeeper-3.4.9/conf/
    # cp zoo_sample.cfg zoo.cfg
    # mkdir /etc/zookeeper
    # ln -sf /usr/local/zookeeper-3.4.9/conf/zoo.cfg /etc/zookeeper/
    # mkdir -p /var/lib/zookeeper
    # chown -R zookeeper:zookeeper /var/lib/zookeeper
    
    # 用来标识主机
    #echo 1 > /var/lib/zookeeper/myid
    #echo 2 > /var/lib/zookeeper/myid 
    #echo 3 > /var/lib/zookeeper/myid

![复制代码][copycode.gif]

# vi /etc/zookeeper/zoo.cfg 
    tickTime=2000
    initLimit=10
    syncLimit=5
    dataDir=/var/lib/zookeeper
    clientPort=2181
    maxClientCnxns=0
    server.1=192.168.190.152:2888:3888
    server.2=192.168.190.153:2888:3888
    server.3=192.168.190.154:2888:3888

![复制代码][copycode.gif]

[root@localhost ~]# vi /etc/init.d/zookeeper 
    #! /bin/sh
    #
    # chkconfig: 2345 90 10
    # description: zookeeper daemon
    
    . /etc/init.d/functions
    
    # You will probably want to change only two following lines.
    BASEDIR="/usr/local/zookeeper-3.4.9"
    USER="zookeeper"
    
    PROG="zookeeper"
    CMD="bin/zkServer.sh"
    RETVAL=0
    
    start () {
    echo -n $"Starting ${PROG}: "
    runuser ${USER} -c "cd ${BASEDIR}
    ${CMD} start > /dev/null &"
    echo
    }
    
    stop () {
    echo -n $"Stopping ${PROG}: "
    runuser ${USER} -c "cd ${BASEDIR}
    ${CMD} stop > /dev/null &"
    echo
    }
    
    restart () {
    stop
    start
    }
    
    
    # See how we were called.
    case "$1" in
    start)
    start
    ;;
    stop)
    stop
    ;;
    restart)
    restart
    ;;
    *)
    echo $"Usage: $0 {start|stop|status}"
    RETVAL=2
    ;;
    esac
    
    exit $RETVAL
    
    # chmod 755 /etc/init.d/zookeeper
    # /etc/init.d/zookeeper start
    # chkconfig zookeeper on

![复制代码][copycode.gif]

# ./zkCli.sh
    
    # 查看当前节点列表
    [zk: localhost:2181(CONNECTED) 6] ls /
    [zookeeper]
    
    # 创建节点
    [zk: localhost:2181(CONNECTED) 7] create /test "test"
    Created /test
    [zk: localhost:2181(CONNECTED) 8] ls /
    [zookeeper, test]
    
    # 查看节点数据
    [zk: localhost:2181(CONNECTED) 9] get /test
    test
    cZxid = 0x100000004
    ctime = Thu Oct 13 08:35:04 CST 2016
    mZxid = 0x100000004
    mtime = Thu Oct 13 08:35:04 CST 2016
    pZxid = 0x100000004
    cversion = 0
    dataVersion = 0
    aclVersion = 0
    ephemeralOwner = 0x0
    dataLength = 4
    numChildren = 0
    
    # 设置节点数据
    [zk: localhost:2181(CONNECTED) 10] set /test "111111"
    cZxid = 0x100000004
    ctime = Thu Oct 13 08:35:04 CST 2016
    mZxid = 0x100000005
    mtime = Thu Oct 13 08:35:41 CST 2016
    pZxid = 0x100000004
    cversion = 0
    dataVersion = 1
    aclVersion = 0
    ephemeralOwner = 0x0
    dataLength = 6
    numChildren = 0
    
    # 删除节点数据
    [zk: localhost:2181(CONNECTED) 11] delete /test
    [zk: localhost:2181(CONNECTED) 12] ls /
    [zookeeper]

![复制代码][copycode.gif]

![777900-20161013230209531-1103806528.png][]

![777900-20161013230714765-1305172348.png][]

**二.kafka集群安装**

\# 环境

192.168.190.152
    192.168.190.153
    192.168.190.154

wget http://mirrors.cnnic.cn/apache/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz
    
    # tar zxvf kafka_2.11-0.10.0.1.tgz
    # mv kafka_2.11-0.10.0.1 /usr/local/
    # cd /usr/local/kafka_2.11-0.10.0.1/

![复制代码][copycode.gif]

# vi config/server.properties 
    broker.id=0 # id唯一 0 1 2
    port=9092
    host.name=192.168.190.152 #192.168.190.153 192.168.190.154
    num.network.threads=3
    num.io.threads=8
    socket.send.buffer.bytes=102400
    socket.receive.buffer.bytes=102400
    socket.request.max.bytes=104857600
    log.dirs=/tmp/kafka-logs
    num.partitions=1
    default.replication.factor=2
    num.recovery.threads.per.data.dir=1
    log.retention.hours=168
    log.segment.bytes=1073741824
    log.retention.check.interval.ms=300000
    zookeeper.connect=192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181
    zookeeper.connection.timeout.ms=6000

![复制代码][copycode.gif]

# useradd kafka
    # touch /etc/init.d/kafka
    # chmod 755 /etc/init.d/kafka

![复制代码][copycode.gif]

# vi /etc/init.d/kafka 
    #!/bin/sh
    #
    # chkconfig: 345 99 01
    # description: Kafka
    #
    # File : Kafka
    #
    # Description: Starts and stops the Kafka server
    #
    
    source /etc/rc.d/init.d/functions
    
    KAFKA_HOME=/usr/local/kafka_2.11-0.10.0.1
    KAFKA_USER=kafka
    # See how we were called.
    case "$1" in
    
    start)
    echo -n "Starting Kafka:"
    /sbin/runuser $KAFKA_USER -c "nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties > /var/log/kafka/server.out 2> /var/log/kafka/server.err &"
    echo " done."
    exit 0
    ;;
    
    stop)
    echo -n "Stopping Kafka: "
    /sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk '{print \$2}' | xargs kill"
    echo " done."
    exit 0
    ;;
    hardstop)
    echo -n "Stopping (hard) Kafka: "
    /sbin/runuser $KAFKA_USER -c "ps -ef | grep kafka.Kafka | grep -v grep | awk '{print \$2}' | xargs kill -9"
    echo " done."
    exit 0
    ;;
    
    status)
    c_pid=`ps -ef | grep kafka.Kafka | grep -v grep | awk '{print $2}'`
    if [ "$c_pid" = "" ] ; then
    echo "Stopped"
    exit 3
    else
    echo "Running $c_pid"
    exit 0
    fi
    ;;
    
    restart)
    stop
    start
    ;;
    
    *)
    echo "Usage: cassandra {start|stop|hardstop|status|restart}"
    exit 1
    ;;
    
    esac

![复制代码][copycode.gif]

# mkdir -p /var/log/kafka
    # chown -R kafka.kafka /var/log/kafka
    # chown -R kafka.kafka /usr/local/kafka_2.11-0.10.0.1/logs
    
    
    # 启动服务
    # /etc/init.d/kafka start
    
    # 创建topic
    # bin/kafka-topics.sh --create --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --replication-factor 2 --partitions 1 --topic test333
    
    # 删除
    # bin/kafka-topics.sh --delete --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333
    
    # 查看topic
    # bin/kafka-topics.sh --list --zookeeper localhost:2181
    
    
    # 开启生产端
    # bin/kafka-console-producer.sh --broker-list 192.168.190.152:9092,192.168.190.153:9092,192.168.190.154:9092 --topic test333
    
    
    # 开启消费端
    # bin/kafka-console-consumer.sh --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333 --from-beginning
    
    
    # bin/kafka-topics.sh --describe --zookeeper 192.168.190.152:2181,192.168.190.153:2181,192.168.190.154:2181 --topic test333

![复制代码][copycode.gif]

\# 错误处理  
\[2016-10-14 06:36:18,401\] WARN Error while fetching metadata with correlation id 0 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:19,543\] WARN Error while fetching metadata with correlation id 1 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:19,680\] WARN Error while fetching metadata with correlation id 2 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:19,908\] WARN Error while fetching metadata with correlation id 3 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:20,116\] WARN Error while fetching metadata with correlation id 4 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:20,334\] WARN Error while fetching metadata with correlation id 5 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:20,505\] WARN Error while fetching metadata with correlation id 6 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)  
\[2016-10-14 06:36:20,757\] WARN Error while fetching metadata with correlation id 7 : \{test333=LEADER\_NOT\_AVAILABLE\} (org.apache.kafka.clients.NetworkClient)

config/server.properties kafka必须配置hostname

![777900-20161013225744078-910066103.png][]

![777900-20161013225752609-1990494208.png][]

![777900-20161013225758578-27322994.png][]

分类: [Kafka][]

[http_www.cnblogs.com_caoguo_p_5958608.html]: http://www.cnblogs.com/caoguo/p/5958608.html
[copycode.gif]: /images/20220404/586e32fbb7a2424d988b324b4b724c31.png
[777900-20161013230209531-1103806528.png]: /images/20220404/9694e1e3985c4792bf5a2b18cbe8521f.png
[777900-20161013230714765-1305172348.png]: /images/20220404/6c49e6386da9449e98140fb00c895b9e.png
[777900-20161013225744078-910066103.png]: /images/20220404/9d5342f6025447f2818ee49f95912183.png
[777900-20161013225752609-1990494208.png]: /images/20220404/1f01b5ba314849998d9732273639107a.png
[777900-20161013225758578-27322994.png]: /images/20220404/cc01c5002faa4b9f96a510d0078fae41.png
[Kafka]: https://www.cnblogs.com/caoguo/category/894520.html