前置准备
kafka前置需要准备
- JDK 1.8+
- Scala
- Zookeeper
部署
wget https://archive.apache.org/dist/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz
tar -xzvf ~/software/kafka_2.11-0.10.0.1.tgz -C ~/app
mkdir -p /home/hadoop/app/tmp/kafka-logs
vi ~/app/kafka_2.11-0.10.0.1/config/server.properties
broker.id=1
port=9092
host.name=hadoop000
log.dirs=/home/hadoop/app/tmp/kafka-logs
log.flush.interval.messages=10000
zookeeper.connect=hadoop000:2181/kafka
启动
nohup ~/app/kafka_2.11-0.10.0.1/bin/kafka-server-start.sh ~/app/kafka_2.11-0.10.0.1/config/server.properties &
常用操作
# 创建topic
~/app/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --create \
--zookeeper hadoop000:2181/kafka \
--replication-factor 1 \
--partitions 1 \
--topic test_topic
# 查看当前topic
~/app/kafka_2.11-0.10.0.1/bin/kafka-topics.sh --list \
--zookeeper hadoop000:2181/kafka
# 启动一个生产者(测试使用)
# 注意:生产者的目标对接是kafka,所以是broker地址
~/app/kafka_2.11-0.10.0.1/bin/kafka-console-producer.sh \
--broker-list hadoop000:9092 \
--topic test_topic
# 启动一个消费者(测试使用)
# 注意:消费者的目标对接是zk,所以是zk地址
~/app/kafka_2.11-0.10.0.1/bin/kafka-console-consumer.sh \
--zookeeper hadoop000:2181/kafka \
--topic test_topic \
--from-beginning