10.0.0.171 kafka01 zookeeper01
10.0.0.172 kafka02 zookeeper02
10.0.0.173 kafka03 zookeeper03
192.168.30.171 kafka01 zookeeper01
192.168.30.172 kafka02 zookeeper02
192.168.30.173 kafka03 zookeeper03
## kafka vm 생성
$ openstack flavor create --id ka1 --ram 8192 --disk 160 --vcpus 2 kafka
$ openstack server create --image 7498cf9d-bd2e-4401-9ae9-ca72120272ed \
--flavor ka1 --nic net-id=03a6de58-9693-4c41-9577-9307c8750141,v4-fixed-ip=10.0.0.171 \
--key-name magnum-key --security-group default kafka01
$ openstack ip floating create --floating-ip-address 192.168.30.171 public
$ openstack ip floating add 192.168.30.171 kafka01
## Oracle Java 8 설치
$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt-get update
$ sudo apt-get install oracle-java8-installer
## 여러 버전을 Java 를 설치했을 때 관리
$ sudo update-alternatives --config java
## zookeeper 설치
## https://zookeeper.apache.org/doc/r3.4.9/zookeeperStarted.html
## http://apache.mirror.cdnetworks.com/zookeeper/zookeeper-3.4.9/
$ mkdir -p downloads && cd downloads
$ wget http://apache.mirror.cdnetworks.com/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz
$ sudo tar -C /usr/local -xzvf zookeeper-3.4.9.tar.gz
$ cd /usr/local
$ sudo ln -s zookeeper-3.4.9/ zookeeper
$ vi /usr/local/zookeeper/conf/zoo.cfg
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=zookeeper01:2888:3888
server.2=zookeeper02:2888:3888
server.3=zookeeper03:2888:3888
$ vi /usr/local/zookeeper/bin/zkEnv.sh
56 ZOO_LOG_DIR="/var/log/zookeeper"
$ sudo mkdir -p /var/log/zookeeper && sudo chown -R stack.stack /var/log/zookeeper
## zookeeper myid 는 서버마다 지정
$ sudo mkdir -p /var/lib/zookeeper && sudo chown -R stack.stack /var/lib/zookeeper
$ vi /var/lib/zookeeper/myid
1
$ vi ~/.bashrc
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export ZOOKEEPER_HOME=/usr/local/zookeeper
PATH=$PATH:$ZOOKEEPER_HOME/bin
$ . ~/.bashrc
$ zkServer.sh start
## zookeeper 설치 확인
$ zkCli.sh -server zookeeper01:2181
## Kafka 설치
## https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-14-04
## https://kafka.apache.org/downloads.html
## https://kafka.apache.org/documentation.html
$ cd downloads
$ wget http://apache.mirror.cdnetworks.com/kafka/0.10.0.1/kafka_2.11-0.10.0.1.tgz
$ sudo tar -C /usr/local -xzvf kafka_2.11-0.10.0.1.tgz
$ cd /usr/local && sudo chown -R stack.stack kafka_2.11-0.10.0.1
$ sudo ln -s kafka_2.11-0.10.0.1/ kafka
## broker id 는 서버마다 고유하게 줘야 함
$ vi /usr/local/kafka/config/server.properties
20 broker.id=0
56 log.dirs=/var/lib/kafka
112 zookeeper.connect=zookeeper01:2181,zookeeper02:2181,zookeeper03:2181
117 delete.topic.enable = true
$ sudo mkdir -p /var/lib/kafka && sudo chown -R stack.stack /var/lib/kafka
$ sudo mkdir -p /var/log/kafka && sudo chown -R stack.stack /var/log/kafka
$ vi ~/.bashrc
export KAFKA_HOME=/usr/local/kafka
PATH=$PATH:$KAFKA_HOME/bin
$ . ~/.bashrc
$ nohup kafka-server-start.sh $KAFKA_HOME/config/server.properties > /var/log/kafka/kafka.log 2>&1 &
## kafkaT : kafka cluster 관리
$ sudo apt-get -y install ruby ruby-dev build-essential
$ sudo gem install kafkat --source https://rubygems.org --no-ri --no-rdoc
$ vi ~/.kafkatcfg
{
"kafka_path": "/usr/local/kafka",
"log_path": "/var/lib/kafka",
"zk_path": "zookeeper01:2181,zookeeper02:2181,zookeeper03:2181"
}
## kafka partition 보기
$ kafkat partitions
## kafka data 테스트
$ echo "Hello, World" | kafka-console-producer.sh --broker-list kafka01:9092,kafka02:9092,kafka03:9092 --topic TutorialTopic > /dev/null
$ kafka-console-consumer.sh --zookeeper zookeeper01:2181,zookeeper02:2181,zookeeper03:2181 --topic TutorialTopic --from-beginning
$ kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
$ kafka-topics.sh --list --zookeeper localhost:2181
$ kafka-console-producer.sh --broker-list localhost:9092 --topic test
This is a message
This is another message
$ kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
## Replica 3 테스트
$ kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic my-replicated-topic
$ kafka-topics.sh --describe --zookeeper localhost:2181 --topic my-replicated-topic
Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs:
Topic: my-replicated-topic Partition: 0 Leader: 0 Replicas: 0,2,1 Isr: 0,2,1
$ kafka-console-producer.sh --broker-list localhost:9092 --topic my-replicated-topic
my test message 1
my test message 2
^C
$ kafka-console-consumer.sh --zookeeper localhost:2181 --from-beginning --topic my-replicated-topic
## 서버 한대 다운
$ kafka-server-stop.sh
$ kafka-topics.sh --describe --zookeeper localhost:2181 --topic my-replicated-topic
Topic:my-replicated-topic PartitionCount:1 ReplicationFactor:3 Configs:
Topic: my-replicated-topic Partition: 0 Leader: 0 Replicas: 0,2,1 Isr: 0,1
$ kafka-console-consumer.sh --zookeeper localhost:2181 --from-beginning --topic my-replicated-topic
## 토픽 삭제
$ kafka-topics.sh --delete --zookeeper localhost:2181 --topic my-replicated-topic
$ kafka-topics.sh --delete --zookeeper localhost:2181 --topic TutorialTopic
