## https://github.com/kubernetes/kubernetes/tree/master/examples/volumes/cephfs

[ ceph-admin 노드에서 ]
$ ssh ceph@192.168.30.22

## kubes Pool 생성
$ ceph osd pool create kubes 128

## kube user 생성
$ ceph auth get-or-create client.kube mon 'allow r' \
osd 'allow class-read object_prefix rbd_children, allow rwx pool=kubes'


[client.kube]
    key = AQCt/BpYigJ7MRAA5vy+cl39EsKpY3C+tXEGrA==

## kube user 에 대한 secret key 생성 및 조회
$ ceph auth get-or-create client.kube
AQCt/BpYigJ7MRAA5vy+cl39EsKpY3C+tXEGrA==


## kube-node01, kube-node02 서버에 kube key 와 ceph.conf 추가
$ ssh stack@192.168.30.15 sudo mkdir -p /etc/ceph
$ ceph auth get-or-create client.kube | ssh stack@192.168.30.15 sudo tee /etc/ceph/ceph.client.kube.keyring
$ cat /etc/ceph/ceph.conf | ssh stack@192.168.30.15 sudo tee /etc/ceph/ceph.conf
$ ssh stack@192.168.30.15 sudo chown -R stack.stack /etc/ceph

$ ssh stack@192.168.30.16 sudo mkdir -p /etc/ceph
$ ceph auth get-or-create client.kube | ssh stack@192.168.30.16 sudo tee /etc/ceph/ceph.client.kube.keyring
$ cat /etc/ceph/ceph.conf | ssh stack@192.168.30.16 sudo tee /etc/ceph/ceph.conf
$ ssh stack@192.168.30.16 sudo chown -R stack.stack /etc/ceph



[ kube-node01, kube-node02 에 접속 ]
## ceph rbd client (ceph-common) 와 ceph fs client 설치 (ceph-fs-common)
$ sudo apt-get -y install ceph-common ceph-fs-common



########################################
## ceph rbd 로 연결하는 방식
########################################

## https://github.com/kubernetes/kubernetes/tree/master/examples/volumes/rbd
## https://github.com/ceph/ceph-docker/tree/master/examples/kubernetes

[ ceph-admin 노드에서 ]

## kube keyring 파일 넣기
$ sudo vi /etc/ceph/ceph.client.kube.keyring
[client.kube]
    key = AQCt/BpYigJ7MRAA5vy+cl39EsKpY3C+tXEGrA==


## rbd 이미지 생성
## http://karan-mj.blogspot.kr/2013/12/ceph-installation-part-3.html

$ rbd create ceph-rbd-test --pool kubes --name client.kube --size 1G -k /etc/ceph/ceph.client.kube.keyring

$ rbd list --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring
$ rbd -p kubes ls


## Jewel 의 새기능은 현재 대부분의 OS 에서 mount 문제가 있어 image 기능을 제거 해야 함
$ rbd feature disable ceph-rbd-test fast-diff --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring
$ rbd feature disable ceph-rbd-test deep-flatten --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring
$ rbd feature disable ceph-rbd-test object-map --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring
$ rbd feature disable ceph-rbd-test exclusive-lock --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring

$ rbd info ceph-rbd-test --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring
$ rbd --image ceph-rbd-test -p kubes info

$ rbd remove ceph-rbd-test --pool kubes --name client.kube -k /etc/ceph/ceph.client.kube.keyring


## secret yaml 을 만들기 위해 key 를 base64 로 인코딩 함
$ grep key /etc/ceph/ceph.client.kube.keyring |awk '{printf "%s", $NF}'|base64
QVFDdC9CcFlpZ0o3TVJBQTV2eStjbDM5RXNLcFkzQyt0WEVHckE9PQ==




[ kube-deploy 접속 ]

## secret key 를 pod 로 생성하여 접속
$ vi ~/kube/ceph-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: ceph-secret
data:
  key: QVFDdC9CcFlpZ0o3TVJBQTV2eStjbDM5RXNLcFkzQyt0WEVHckE9PQ==

$ scp ~/kube/ceph-secret.yaml kube-master01:~/kube/.
$ ssh kube-master01 "kubectl create -f ~/kube/ceph-secret.yaml"
$ kubectl -s http://kube-master01:8080 get secrets


## rbd-with-secret pod 생성해서 rbd 활용
$ vi ~/kube/rbd-with-secret.yml
apiVersion: v1
kind: Pod
metadata:
  name: rbd2
spec:
  containers:
  - image: gcr.io/google_containers/busybox
    command:
    - sleep
    - "3600"
    imagePullPolicy: IfNotPresent
    name: rbd-rw-busybox
    volumeMounts:
    - mountPath: "/mnt/rbd"
      name: rbdpd
  volumes:
  - name: rbdpd
    rbd:
      monitors:
      - 192.168.30.23:6789
      - 192.168.30.24:6789
      - 192.168.30.25:6789
      pool: kubes
      image: ceph-rbd-test
      user: kube
      keyring: /etc/ceph/ceph.client.kube.keyring
      secretRef:
        name: ceph-secret
      fsType: ext4
      readOnly: false


$ scp ~/kube/rbd-with-secret.yml kube-master01:~/kube/.
$ ssh kube-master01 "kubectl create -f ~/kube/rbd-with-secret.yml"
$ kubectl -s http://kube-master01:8080 get pods




## rbd 연결 확인
$ kubectl -s http://kube-master01:8080 describe pods rbd2
$ kubectl -s http://kube-master01:8080 exec -it rbd2 -- df -h



[ kube-node02 접속하여 ]

$ docker ps
$ docker inspect --format '{{ .Mounts }}' 4c4070a1393b

## 혹은
$ mount |grep kub
/dev/rbd0 on /var/lib/kubelet/plugins/kubernetes.io/rbd/rbd/kubes-image-ceph-rbd-test type ext4 (rw,relatime,stripe=1024,data=ordered)
/dev/rbd0 on /var/lib/kubelet/pods/061973fc-a265-11e6-940f-5cb9018c67dc/volumes/kubernetes.io~rbd/rbdpd type ext4 (rw,relatime,stripe=1024,data=ordered)




[ kube-deploy 접속해서 ]

## secret key pod 를 사용하지 않고 keyring 으로만 rbd pod 생성
$ vi ~/kube/rbd.yml
apiVersion: v1
kind: Pod
metadata:
  name: rbd
spec:
  containers:
  - image: gcr.io/google_containers/busybox
    command:
    - sleep
    - "3600"
    imagePullPolicy: IfNotPresent
    name: rbd-rw-busybox
    volumeMounts:
    - mountPath: "/mnt/rbd"
      name: rbdpd
  volumes:
  - name: rbdpd
    rbd:
      monitors:
      - 192.168.30.23:6789
      - 192.168.30.24:6789
      - 192.168.30.25:6789
      pool: kubes
      image: ceph-rbd-test
      user: kube
      keyring: /etc/ceph/ceph.client.kube.keyring
      fsType: ext4
      readOnly: false


$ scp ~/kube/rbd.yml kube-master01:~/kube/.
$ ssh kube-master01 "kubectl create -f ~/kube/rbd.yml"
$ kubectl -s http://kube-master01:8080 get pods

## rbd 연결 확인

$ kubectl -s http://kube-master01:8080 exec -it rbd -- df -h 


Posted by Kubernetes Korea co-leader seungkyua@gmail.com