안승규의 블로그 (Stay hungry, stay foolish)

1. sudo 사용자 추가, 보안 s/w 내리기, swap off

$ sudo adduser ask

$ cat <<EOF | sudo tee /etc/sudoers.d/sudoers-ask
ask     ALL=(ALL:ALL)   NOPASSWD:ALL
EOF

$ sudo systemctl stop ufw
$ sudo systemctl disable ufw
$ sudo systemctl stop apparmor.service
$ sudo systemctl disable apparmor.service
$ sudo swapoff -a

2. module load

$ cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

$ sudo modprobe overlay
$ sudo modprobe br_netfilter

3. network forwarding 설정

$ cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF


$ sudo sysctl --system
$ sudo iptables -P FORWARD ACCEPT

4. containerd 설치

$ sudo apt-get update
$ sudo apt-get install -y apt-transport-https ca-certificates curl gpg
$ sudo install -m 0755 -d /etc/apt/keyrings
$ sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
$ sudo chmod a+r /etc/apt/keyrings/docker.asc

$ echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

$ sudo apt-get update
$ sudo apt-get install -y containerd.io

$ sudo mkdir -p /etc/containerd
$ sudo containerd config default | sudo tee /etc/containerd/config.toml

$ sudo sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml

$ sudo systemctl restart containerd
$ sudo systemctl enable containerd
$ sudo systemctl status containerd

5. crictl 설치

#----------------------------------------------------------
# 서버가 arm64 or amd64 인지 확인하여 설치
#----------------------------------------------------------
$ VERSION="v1.30.0"
$ curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-arm64.tar.gz --output crictl-${VERSION}-linux-arm64.tar.gz

$ sudo tar zxvf crictl-$VERSION-linux-arm64.tar.gz -C /usr/local/bin
$ rm -f crictl-$VERSION-linux-arm64.tar.gz

#----------------------------------------------------------
# crictl 이 어느 container 를 접속할 것인지 세팅
#----------------------------------------------------------
$ cat <<EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 2
debug: false
pull-image-on-create: false
EOF

$ sudo bash -c "crictl completion > /etc/bash_completion.d/crictl"
$ source ~/.bashrc


#----------------------------------------------------------
# containerd 설정 확인
#----------------------------------------------------------
$ sudo crictl info

6. kubectl 설치

#----------------------------------------------------------
# 서버가 arm64 or amd64 인지 확인하여 설치
#----------------------------------------------------------
$ curl -LO "https://dl.k8s.io/release/v1.30.0/bin/linux/arm64/kubectl"
$ chmod +x ./kubectl
$ sudo mv ./kubectl /usr/local/bin/kubectl

7.  Kubernetes 설치

$ mkdir -p ~/kubeadm && cd ~/kubeadm

#-----------------------------------------------
# kubernetes 다운로드 key 와 url 등록
#-----------------------------------------------
$ curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

$ echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
$ sudo apt-get update


#========================================================================
# kubelet 설치 (아직 kubelet 이 뜨지는 않음)
#========================================================================
$ sudo apt-get install -y kubelet="1.30.3-*" kubeadm="1.30.3-*"
$ sudo systemctl enable --now kubelet
$ sudo systemctl start kubelet

#========================================================================
# 1. kubeadm 설치
#========================================================================
$ sudo kubeadm config images pull

#------------------------------------------------------------
# 자기 노드의 ip: --apiserver-advertise-address
# multi control-plane 일 경우 L4 ip: --control-plane-endpoint
# cgroup driver 세팅 (https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/)
#------------------------------------------------------------
$ vi kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
nodeRegistration:
  criSocket: "/var/run/containerd/containerd.sock"

---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
  certSANs:
  - 127.0.0.1
  - localhost
  - <Node private IP>
  - <Node public IP>
networking:
  serviceSubnet: 10.233.0.0/18
  podSubnet: 10.233.64.0/18
  dnsDomain: "cluster.local"


#----------------------------------------------------------------
# kubeadm init 을 하고 나면 /var/lib/kubelet/config.yaml 이 생성되어
# kubelet 이 정상적으로 실행됨
#----------------------------------------------------------------
$ sudo kubeadm init --config kubeadm-config.yaml --v=5


#------------------------------------------------------------
# kubeconfig
#------------------------------------------------------------
$ mkdir -p ~/.kube
$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

8. Calico 설치

$ mkdir -p ~/calico && cd ~/calico

$ curl -LO https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/tigera-operator.yaml

$ kubectl create -f tigera-operator.yaml

$ curl -LO https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/custom-resources.yaml

#------------------------------------------------------------
# yaml 을 열어서 pod 네트워크를 확인하고 변경
#------------------------------------------------------------
$ vi custom-resources.yaml
...
    cidr: 10.233.64.0/18
...

$ kubectl create -f custom-resources.yaml

#------------------------------------------------------------
# calico 설치 확인
#------------------------------------------------------------
$ kubectl get pods -n calico-system

9. kubectl bash completion

$ source <(kubectl completion bash)
$ kubectl completion bash > ~/.kube/completion.bash.inc

$ printf "
# kubectl shell completion
source '$HOME/.kube/completion.bash.inc'
" >> $HOME/.bash_aliases

$ source $HOME/.bash_aliases

10. taint 제거

$ kubectl taint nodes --all node-role.kubernetes.io/control-plane-

11. 설치 테스트

$ mkdir -p ~/sample-yaml && cd ~/sample-yaml

$ cat <<EOF | tee ./nginx-service.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx-deployment
  name: nginx-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.21.0
        ports:
        - containerPort: 80

---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service-nodeport
spec:
  selector:
    app: nginx
  ports:
  - protocol: TCP
    port: 8080
    targetPort: 80
    nodePort: 30180
  type: NodePort
  externalTrafficPolicy: Local
EOF


$ kubectl apply -f nginx-service.yaml

$ kubectl get pods

$ curl <Node ip>:30180