0. 서버 설정

Master   : 192.168.75.211  (etcd, kube-apiserver, kube-controller-manager, kube-scheduler)

Node01  : 192.168.75.212  (kube-proxy, kubelet)

Node02  : 192.168.75.213  (kube-proxy, kubelet)


etcd-2.2.1, flannel-0.5.5, k8s-1.1.2



[ Master Node 서버에 모두 설치 ]

1. apt-get 으로 필요 s/w 설치

# docker 설치

$ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D

$ sudo vi /etc/apt/sources.list.d/docker.list


# Debian Jessie

#deb https://apt.dockerproject.org/repo debian-jessie main


# Debian Stretch/Sid

#deb https://apt.dockerproject.org/repo debian-stretch main


# Ubuntu Precise

#deb https://apt.dockerproject.org/repo ubuntu-precise main


# Ubuntu Trusty (14.04 LTS)

deb https://apt.dockerproject.org/repo ubuntu-trusty main


# Ubuntu Utopic (14.10)

#deb https://apt.dockerproject.org/repo ubuntu-utopic main


# Ubuntu Vivid (15.04)

#deb https://apt.dockerproject.org/repo ubuntu-vivid main


# Ubuntu Wily (15.10)

#deb https://apt.dockerproject.org/repo ubuntu-wily main


$ sudo apt-get update

$ sudo apt-get purge lxc-docker*

$ sudo apt-get purge docker.io

$ sudo apt-get autoremove

$ sudo apt-get install docker-engine


$ sudo apt-get install bridge-utils

sudo usermod -a -G docker stack      # stack user에 docker 그룹을 추가

$ sudo service docker restart



2. sudo 세팅

# gpasswd -a stack sudo   (이건 안되는데??)

stack   ALL=(ALL:ALL) NOPASSWD: ALL



3. ntp 설치 & ssh 키 설치

# ssh 로 master <-> Node 사이에 stack 계정으로 바로 접속할 수 있어야 함

# ssh 로 master, Node 각각 자기 서버 내에서 stack 계정에서 root 계정으로 바로 접속할 수 있어야 함



4. host 세팅

192.168.75.211    master

192.168.75.212    node01

192.168.75.213    node02



5. Go 설치

1. 다운로드

$ cd /home/stack/downloads

wget https://storage.googleapis.com/golang/go1.5.2.linux-amd64.tar.gz

sudo tar -C /usr/local -xzf go1.5.2.linux-amd64.tar.gz


2. 환경변수 세팅

sudo vi /etc/profile

export GOROOT=/usr/local/go

export PATH=$PATH:/usr/local/go/bin


sudo visudo             # sudo 에서도 go path가 적용될려면 여기에 세팅

Defaults    env_reset

Defaults    env_keep += "GOPATH"

Defaults        secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin"


$ cd

vi .bash_profile

export GOPATH=$HOME/Documents/go_workspace:$HOME/Documents/go_workspace/src/k8s.io/kubernetes/Godeps/_workspace

export PATH=$HOME/Documents/go_workspace/bin:$PATH



6. kubernetes 설치

# go 로 다운로드하기

$ go get k8s.io/kubernetes   # git clone https://github.com/kubernetes/kubernetes.git


$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes

$ git checkout -b v1.1.2 tags/v1.1.2

$ make all                                      # _output 디렉토리에 결과 파일이 생성


# 소스 수정 후 make 로 재빌드 (참고)   _output 디렉토리에 결과 파일이 생성

$ make all WHAT=plugin/cmd/kube-scheduler GOFLAGS=-v      # scheduler

$ make all WHAT=cmd/kubelet GOFLAGS=-v                           # kubelet

$ make all WHAT=cmd/kube-apiserver GOFLAGS=-v                # apiserver


# 소스 수정 후 재빌드 (참고)

$ hack/build-go.sh                  # make를 돌리면 build-go.sh 가 수행됨
$ hack/local-up-cluster.sh        # 로컬 클러스터를 생성할 때


$ sudo su -

# cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu


# export KUBE_VERSION=1.1.2

# export FLANNEL_VERSION=0.5.5

# export ETCD_VERSION=2.2.1


# ./build.sh                 # binaries 디렉토리에 다운 받음

# exit



$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu

$ vi config-default.sh


export nodes="stack@192.168.75.211 stack@192.168.75.212"

export role="a i"

export NUM_MINIONS=${NUM_MINIONS:-1}

export SERVICE_CLUSTER_IP_RANGE=192.168.230.0/24

export FLANNEL_NET=172.16.0.0/16



ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"

DNS_SERVER_IP=${DNS_SERVER_IP:-"192.168.230.10"}

DNS_DOMAIN="cluster.local"

DNS_REPLICAS=1


ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"


$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster

$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh


# 복사한 파일

make-ca-cert.sh    

reconfDocker.sh    

config-default.sh    

util.sh    

kube-scheduler.conf    

kube-apiserver.conf    

etcd.conf    

kube-controller-manager.conf    

flanneld.conf    

kube-controller-manager    

kube-scheduler    

etcd    

kube-apiserver    

flanneld    

kube-controller-manager    

etcdctl    

kube-scheduler    

etcd    

kube-apiserver    

flanneld



# kubectl 복사

$ sudo cp ubuntu/binaries/kubectl /opt/bin/.


# 경로 추가

$ vi ~/.bash_profile

export PATH=/opt/bin:$PATH

export KUBECTL_PATH=/opt/bin/kubectl



# Add-on 설치

$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu

$ KUBERNETES_PROVIDER=ubuntu ./deployAddons.sh


# 에러 발생하면 아래 실행 (Docker image 를 다운로드 함)

$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes

./build/run.sh hack/build-cross.sh


# Add-on 설치 다시

cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cluster/ubuntu

$ KUBERNETES_PROVIDER=ubuntu ./deployAddons.sh



[ Kubernetes 설치 지우기 ]

$ cd ..

$ KUBERNETES_PROVIDER=ubuntu ./kube-down.sh


# node01 에 떠 있는 docker 삭제하기

docker ps -a | awk '{print $1}' | xargs docker stop

docker ps -a | awk '{print $1}' | xargs docker rm

$ sudo cp ubuntu/binaries/kubectl /opt/bin/.                # kubectl 을 /opt/bin 에 복사해야 함


$ KUBERNETES_PROVIDER=ubuntu ./kube-up.sh



[ Master의 Docker를 flannel 로 연결 ]

sudo service docker stop

$ sudo ip link set dev docker0 down

$ sudo brctl delbr docker0

$ cat /run/flannel/subnet.env      # flannel의 subnet 과 mtu 값을 확인한다.

$ sudo vi /etc/default/docker

DOCKER_OPTS=" -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock --bip=172.16.25.1/24 --mtu=1472"


$ sudo service docker start

$ sudo ip link set dev docker0 up



# node01 에서 docker ps -a 로 가비지가 많이 쌓임. 지워주면 됨

# ssh node01 로 접속하여 가비지 조회

docker ps -a | grep Exited | awk '{print $1}'

docker ps -a | grep Exited | awk '{print $1}' | xargs docker rm


# kubernetes volume 생성되는 곳 : /var/lib/kubelet/pods

# kubernetes garbage-collection https://github.com/kubernetes/kubernetes/blob/master/docs/admin/garbage-collection.md


$ kubectl get nodes

$ kubectl get pods --namespace=kube-system         # add-on pods 확인

$ kubectl cluster-info


# Skydns Pod 정보 보기

kubectl describe pod kube-dns-v9-549av --namespace=kube-system


# DNS 확인

$ kubectl create -f busybox.yaml


$ vi busybox.yaml

apiVersion: v1

kind: Pod

metadata:

  name: busybox

  namespace: default

spec:

  containers:

  - image: busybox

    command:

      - sleep

      - "3600"

    imagePullPolicy: IfNotPresent

    name: busybox

  restartPolicy: Always


$ kubectl get pods busybox


kubectl exec Pod명 [-c Container명] -i -t -- COMMAND [args..] [flags]

$ kubectl exec busybox -- nslookup kubernetes.default


# busybox 삭제하기

$ kubectl delete -f busybox.yaml



# 웹화면 확인

http://192.168.75.211:8080/


# UI 확인

http://192.168.75.211:8080/ui    >> 아래 화면으로 리다이렉션 됨

http://192.168.75.211:8080/api/v1/proxy/namespaces/kube-system/services/kube-ui



# Mac 에서 소스 개발하고 Master 에 소스 커밋하기  (참고)

# 원격에 tag 에도 v1.1.2 가 있고 branch 에도 v1.1.2 가 있으면 remote branch 를 지정

# git push [저장소] (local branch명:)remote branch명

git push origin refs/heads/v1.1.2


git config --global user.name "Seungkyu Ahn" 

git config --global user.email "seungkyua@gmail.com"


# 로컬 파일을 Master 서버로 복사

$ vi ~/bin/cmaster.sh

#!/bin/bash


function change_directory {

  cd /Users/ahnsk/Documents/go_workspace/src/k8s.io/kubernetes

}


change_directory

files=$(git status | grep -E 'modified|new file' | awk -F':' '{print$2}')


for file in $files; do

    scp $file stack@192.168.230.211:/home/stack/Documents/go_workspace/src/k8s.io/kubernetes/$file

done



# kube-apiserver 소스로 띄우기

$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cmd/kube-apiserver


sudo -E go run apiserver.go --insecure-bind-address=0.0.0.0 --insecure-port=8080 --etcd-servers=http://127.0.0.1:4001 --logtostderr=true --service-cluster-ip-range=192.168.230.0/24 --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,DenyEscalatingExec,SecurityContextDeny --service-node-port-range=30000-32767 --client-ca-file=/srv/kubernetes/ca.crt --tls-cert-file=/srv/kubernetes/server.cert --tls-private-key-file=/srv/kubernetes/server.key 



# Document 만들기

$ cd ~/Documents/go_workspace/src/k8s.io/kubernetes/cmd/genkubedocs

$ mkdir -p temp

$  go run gen_kube_docs.go temp kube-apiserver



7. Sample App 올려보기

https://github.com/kubernetes/kubernetes/tree/master/examples/guestbook


# 디렉토리 위치는 kubernetes 설치한 위치

$ sudo kubectl create -f examples/guestbook/redis-master-controller.yaml

$ sudo kubectl get rc

$ sudo kubectl get pods

$ sudo kubectl describe pods/redis-master-xssrd

$ sudo kubectl logs <pod_name>          # container log 확인


$ sudo kubectl create -f examples/guestbook/redis-master-service.yaml

$ sudo kubectl get services


$ sudo kubectl create -f examples/guestbook/redis-slave-controller.yaml

$ sudo kubectl get rc

$ sudo kubectl get pods


$ sudo kubectl create -f examples/guestbook/redis-slave-service.yaml

$ sudo kubectl get services



$ sudo kubectl create -f examples/guestbook/frontend-controller.yaml

$ sudo kubectl get rc

$ sudo kubectl get pods



$ sudo kubectl create -f examples/guestbook/frontend-service.yaml

$ sudo kubectl get services





sudo kubectl describe services frontend

$ sudo kubectl get ep


# dns 보기

$ sudo kubectl get services kube-dns --namespace=kube-system


# 환경변수 보기

$ sudo kubectl get pods -o json

$ sudo kubectl get pods -o wide

$ sudo kubectl exec frontend-cyite -- printenv | grep SERVICE


8. Sample App 삭제

$ sudo kubectl stop rc -l "name in (redis-master, redis-slave, frontend)"

$ sudo kubectl delete service -l "name in (redis-master, redis-slave, frontend)"



# Network

TAP : vm과 eth0 (physical port) 와 연결할 때 사용. tap <-> bridge <-> eth0 로 됨

VETH : docker <-> bridge,  docker <-> OVS, bridge <-> OVS 를 연결할 때 사용


# interconnecting namespaces

http://www.opencloudblog.com/?p=66



# Docker <-> veth 알아내기

$ vi veth.sh


#!/bin/bash


set -o errexit

set -o nounset

#set -o pipefail


VETHS=`ifconfig -a | grep "Link encap" | sed 's/ .*//g' | grep veth`

DOCKERS=$(docker ps -a | grep Up | awk '{print $1}')


for VETH in $VETHS

do

  PEER_IFINDEX=`ethtool -S $VETH 2>/dev/null | grep peer_ifindex | sed 's/ *peer_ifindex: *//g'`

  for DOCKER in $DOCKERS

  do

    PEER_IF=`docker exec $DOCKER ip link list 2>/dev/null | grep "^$PEER_IFINDEX:" | awk '{print $2}' | sed 's/:.*//g'`

    if [ -z "$PEER_IF" ]; then

      continue

    else

      printf "%-10s is paired with %-10s on %-20s\n" $VETH $PEER_IF $DOCKER

      break

    fi

  done

done






Posted by Kubernetes Korea co-leader seungkyua@gmail.com