0. 서버 설정
Master : 192.168.75.211 (etcd, kube-apiserver, kube-controller-manager, kube-scheduler)
Node01 : 192.168.75.212 (kube-proxy, kubelet)
Node02 : 192.168.75.213 (kube-proxy, kubelet)
etcd-2.2.1, flannel-0.5.5, k8s-1.1.2
[ Master Node 서버에 모두 설치 ]
1. apt-get 으로 필요 s/w 설치
# docker 설치
$ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
$ sudo vi /etc/apt/sources.list.d/docker.list
# Debian Jessie
#deb https://apt.dockerproject.org/repo debian-jessie main
# Debian Stretch/Sid
#deb https://apt.dockerproject.org/repo debian-stretch main
# Ubuntu Precise
#deb https://apt.dockerproject.org/repo ubuntu-precise main
# Ubuntu Trusty (14.04 LTS)
deb https://apt.dockerproject.org/repo ubuntu-trusty main
# Ubuntu Utopic (14.10)
#deb https://apt.dockerproject.org/repo ubuntu-utopic main
# Ubuntu Vivid (15.04)
#deb https://apt.dockerproject.org/repo ubuntu-vivid main
# Ubuntu Wily (15.10)
#deb https://apt.dockerproject.org/repo ubuntu-wily main
$ sudo apt-get update
$ sudo apt-get purge lxc-docker*
$ sudo apt-get purge docker.io
$ sudo apt-get autoremove
$ sudo apt-get install docker-engine
$ sudo apt-get install bridge-utils
$ sudo usermod -a -G docker stack # stack user에 docker 그룹을 추가
$ sudo service docker restart
2. sudo 세팅
# gpasswd -a stack sudo (이건 안되는데??)
stack ALL=(ALL:ALL) NOPASSWD: ALL
3. ntp 설치 & ssh 키 설치
# ssh 로 master <-> Node 사이에 stack 계정으로 바로 접속할 수 있어야 함
# ssh 로 master, Node 각각 자기 서버 내에서 stack 계정에서 root 계정으로 바로 접속할 수 있어야 함4. host 세팅
192.168.75.211 master
192.168.75.212 node01
192.168.75.213 node02
5. Go 설치
1. 다운로드
$ cd /home/stack/downloads
$ wget https://storage.googleapis.com/golang/go1.5.2.linux-amd64.tar.gz
$ sudo tar -C /usr/local -xzf go1.5.2.linux-amd64.tar.gz
2. 환경변수 세팅
$ sudo vi /etc/profile
export GOROOT=/usr/local/go
export PATH=$PATH:/usr/local/go/bin
$ sudo visudo # sudo 에서도 go path가 적용될려면 여기에 세팅
Defaults env_reset
Defaults env_keep += "GOPATH"
Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin"
$ cd
$ vi .bash_profile
export GOPATH=$HOME/Documents/go_workspace:$HOME/Documents/go_workspace/src/k8s.io/kubernetes/Godeps/_workspace
export PATH=$HOME/Documents/go_workspace/bin:$PATH
6. kubernetes 설치
# go 로 다운로드하기
$ go get k8s.io/kubernetes # git clone https://github.com/kubernetes/kubernetes.git
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes
$ git checkout -b v1.1.2 tags/v1.1.2
$ make all # _output 디렉토리에 결과 파일이 생성
# 소스 수정 후 make 로 재빌드 (참고) _output 디렉토리에 결과 파일이 생성
$ make all WHAT=plugin/cmd/kube-scheduler GOFLAGS=-v # scheduler
$ make all WHAT=cmd/kubelet GOFLAGS=-v # kubelet
$ make all WHAT=cmd/kube-apiserver GOFLAGS=-v # apiserver
# 소스 수정 후 재빌드 (참고)
$ hack/build-go.sh # make를 돌리면 build-go.sh 가 수행됨
$ hack/local-up-cluster.sh # 로컬 클러스터를 생성할 때
$ sudo su -
# cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu
# export KUBE_VERSION=1.1.2
# export FLANNEL_VERSION=0.5.5
# export ETCD_VERSION=2.2.1
# ./build.sh # binaries 디렉토리에 다운 받음
# exit
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu
$ vi config-default.sh
export nodes="stack@192.168.75.211 stack@192.168.75.212"
export role="a i"
export NUM_MINIONS=${NUM_MINIONS:-1}
export SERVICE_CLUSTER_IP_RANGE=192.168.230.0/24
export FLANNEL_NET=172.16.0.0/16
ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
DNS_SERVER_IP=${DNS_SERVER_IP:-"192.168.230.10"}
DNS_DOMAIN="cluster.local"
DNS_REPLICAS=1
ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster
$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh
# 복사한 파일
make-ca-cert.sh
reconfDocker.sh
config-default.sh
util.sh
kube-scheduler.conf
kube-apiserver.conf
etcd.conf
kube-controller-manager.conf
flanneld.conf
kube-controller-manager
kube-scheduler
etcd
kube-apiserver
flanneld
kube-controller-manager
etcdctl
kube-scheduler
etcd
kube-apiserver
flanneld
# kubectl 복사
$ sudo cp ubuntu/binaries/kubectl /opt/bin/.
# 경로 추가
$ vi ~/.bash_profile
export PATH=/opt/bin:$PATH
export KUBECTL_PATH=/opt/bin/kubectl
# Add-on 설치
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu
$ KUBERNETES_PROVIDER=ubuntu ./deployAddons.sh
# 에러 발생하면 아래 실행 (Docker image 를 다운로드 함)
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes
$ ./build/run.sh hack/build-cross.sh
# Add-on 설치 다시
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu
$ KUBERNETES_PROVIDER=ubuntu ./deployAddons.sh
[ Kubernetes 설치 지우기 ]
$ cd ..
$ KUBERNETES_PROVIDER=ubuntu ./kube-down.sh
# node01 에 떠 있는 docker 삭제하기
$ docker ps -a | awk '{print $1}' | xargs docker stop
$ docker ps -a | awk '{print $1}' | xargs docker rm
$ sudo cp ubuntu/binaries/kubectl /opt/bin/. # kubectl 을 /opt/bin 에 복사해야 함
$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh
[ Master의 Docker를 flannel 로 연결 ]
$ sudo service docker stop
$ sudo ip link set dev docker0 down
$ sudo brctl delbr docker0
$ cat /run/flannel/subnet.env # flannel의 subnet 과 mtu 값을 확인한다.
$ sudo vi /etc/default/docker
DOCKER_OPTS=" -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock --bip=172.16.25.1/24 --mtu=1472"
$ sudo service docker start
$ sudo ip link set dev docker0 up
# node01 에서 docker ps -a 로 가비지가 많이 쌓임. 지워주면 됨
# ssh node01 로 접속하여 가비지 조회
$ docker ps -a | grep Exited | awk '{print $1}'
$ docker ps -a | grep Exited | awk '{print $1}' | xargs docker rm
# kubernetes volume 생성되는 곳 : /var/lib/kubelet/pods
# kubernetes garbage-collection https://github.com/kubernetes/kubernetes/blob/master/docs/admin/garbage-collection.md
$ kubectl get nodes
$ kubectl get pods --namespace=kube-system # add-on pods 확인
$ kubectl cluster-info
# Skydns Pod 정보 보기
$ kubectl describe pod kube-dns-v9-549av --namespace=kube-system
# DNS 확인
$ kubectl create -f busybox.yaml
$ vi busybox.yaml
apiVersion: v1
kind: Pod
metadata:
name: busybox
namespace: default
spec:
containers:
- image: busybox
command:
- sleep
- "3600"
imagePullPolicy: IfNotPresent
name: busybox
restartPolicy: Always
$ kubectl get pods busybox
# kubectl exec Pod명 [-c Container명] -i -t -- COMMAND [args..] [flags]
$ kubectl exec busybox -- nslookup kubernetes.default
# busybox 삭제하기
$ kubectl delete -f busybox.yaml
# 웹화면 확인
http://192.168.75.211:8080/
# UI 확인
http://192.168.75.211:8080/ui >> 아래 화면으로 리다이렉션 됨
http://192.168.75.211:8080/api/v1/proxy/namespaces/kube-system/services/kube-ui
# Mac 에서 소스 개발하고 Master 에 소스 커밋하기 (참고)
# 원격에 tag 에도 v1.1.2 가 있고 branch 에도 v1.1.2 가 있으면 remote branch 를 지정
# git push [저장소] (local branch명:)remote branch명
$ git push origin refs/heads/v1.1.2
$ git config --global user.name "Seungkyu Ahn"
$ git config --global user.email "seungkyua@gmail.com"
# 로컬 파일을 Master 서버로 복사
$ vi ~/bin/cmaster.sh
#!/bin/bash
function change_directory {
cd /Users/ahnsk/Documents/go_workspace/src/k8s.io/kubernetes
}
change_directory
files=$(git status | grep -E 'modified|new file' | awk -F':' '{print$2}')
for file in $files; do
scp $file stack@192.168.230.211:/home/stack/Documents/go_workspace/src/k8s.io/kubernetes/$file
done
# kube-apiserver 소스로 띄우기
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cmd/kube-apiserver
$ sudo -E go run apiserver.go --insecure-bind-address=0.0.0.0 --insecure-port=8080 --etcd-servers=http://127.0.0.1:4001 --logtostderr=true --service-cluster-ip-range=192.168.230.0/24 --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,DenyEscalatingExec,SecurityContextDeny --service-node-port-range=30000-32767 --client-ca-file=/srv/kubernetes/ca.crt --tls-cert-file=/srv/kubernetes/server.cert --tls-private-key-file=/srv/kubernetes/server.key
# Document 만들기
$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cmd/genkubedocs
$ mkdir -p temp
$ go run gen_kube_docs.go temp kube-apiserver
7. Sample App 올려보기
https://github.com/kubernetes/kubernetes/tree/master/examples/guestbook
# 디렉토리 위치는 kubernetes 설치한 위치
$ sudo kubectl create -f examples/guestbook/redis-master-controller.yaml
$ sudo kubectl get rc
$ sudo kubectl get pods
$ sudo kubectl describe pods/redis-master-xssrd
$ sudo kubectl logs <pod_name> # container log 확인
$ sudo kubectl create -f examples/guestbook/redis-master-service.yaml
$ sudo kubectl get services
$ sudo kubectl create -f examples/guestbook/redis-slave-controller.yaml
$ sudo kubectl get rc
$ sudo kubectl get pods
$ sudo kubectl create -f examples/guestbook/redis-slave-service.yaml
$ sudo kubectl get services
$ sudo kubectl create -f examples/guestbook/frontend-controller.yaml
$ sudo kubectl get rc
$ sudo kubectl get pods
$ sudo kubectl create -f examples/guestbook/frontend-service.yaml
$ sudo kubectl get services
$ sudo kubectl describe services frontend
$ sudo kubectl get ep
# dns 보기
$ sudo kubectl get services kube-dns --namespace=kube-system
# 환경변수 보기
$ sudo kubectl get pods -o json
$ sudo kubectl get pods -o wide
$ sudo kubectl exec frontend-cyite -- printenv | grep SERVICE
8. Sample App 삭제
$ sudo kubectl stop rc -l "name in (redis-master, redis-slave, frontend)"
$ sudo kubectl delete service -l "name in (redis-master, redis-slave, frontend)"
# Network
TAP : vm과 eth0 (physical port) 와 연결할 때 사용. tap <-> bridge <-> eth0 로 됨
VETH : docker <-> bridge, docker <-> OVS, bridge <-> OVS 를 연결할 때 사용
# interconnecting namespaces
# Docker <-> veth 알아내기
$ vi veth.sh
#!/bin/bash
set -o errexit
set -o nounset
#set -o pipefail
VETHS=`ifconfig -a | grep "Link encap" | sed 's/ .*//g' | grep veth`
DOCKERS=$(docker ps -a | grep Up | awk '{print $1}')
for VETH in $VETHS
do
PEER_IFINDEX=`ethtool -S $VETH 2>/dev/null | grep peer_ifindex | sed 's/ *peer_ifindex: *//g'`
for DOCKER in $DOCKERS
do
PEER_IF=`docker exec $DOCKER ip link list 2>/dev/null | grep "^$PEER_IFINDEX:" | awk '{print $2}' | sed 's/:.*//g'`
if [ -z "$PEER_IF" ]; then
continue
else
printf "%-10s is paired with %-10s on %-20s\n" $VETH $PEER_IF $DOCKER
break
fi
done
done
