DEVOCEAN

# 외부 패키지 및 이미지를 가져오기 위해 Outbound Any Open 오픈된 상태
 
# swap 제거
sudo swapoff -a
sudo sed -i 's/\/dev\/mapper\/rhel-swap/#\/dev\/mapper\/rhel-swap/g' /etc/fstab
 
# 데이터센터 표준 보안설정 해제 /etc/modprobe.d/blacklist.conf
 
sudo sed -i 's/install nf_conntrack/#install nf_conntrack/g' /etc/modprobe.d/blacklist.conf
sudo sed -i 's/install nf_nat/#install nf_nat/g' /etc/modprobe.d/blacklist.conf
sudo sed -i 's/install nf_defrag_ipv4/#install nf_defrag_ipv4/g' /etc/modprobe.d/blacklist.conf
 
# kubeernetes 레포지토리 추가
# packages.cloud.google.com 서비스 종료로 pkgs.k8s.io 레포 활용 (https://kubernetes.io/blog/2023/08/15/pkgs-k8s-io-introduction/)
sudo cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF
sudo yum clean all
 
 
# overlayFS 파일시스템과 bridge 네트워크 인터페이스를 이용하기 위해 커널 모듈 활성화
echo -e "overlay\nbr_netfilter" | sudo tee /etc/modules-load.d/k8s.conf
# 데이터센터 표준 보안설정으로 인한 modeprobe를 sudo로 사용 할 수 없어 아래와 같이 우회 처리
#sudo ln -s /bin/kmod ./modprobe
#sudo ./modprobe br_netfilter
#sudo ./modprobe overlay
#sudo lsmod
#sudo rm -f ./modprobe
sudo modprobe br_netfilter
 
# 컨테이너의 네트워크 패킷이 호스트머신의 iptables 설정에 따라 제어되도록 하고자 함
echo "net.bridge.bridge-nf-call-iptables = 1" | sudo tee -a /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-ip6tables = 1" | sudo tee -a /etc/sysctl.conf
 
# 패킷을 forward하기 위해 커널 파라미터 변경
sudo sed -i 's/net.ipv4.ip_forward = 0/net.ipv4.ip_forward = 1/g' /etc/sysctl.conf
sudo sysctl -p

# 패키지 설치
sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable kubelet.service
 
# K8s 최신 버전에서는 Docker을 이용하지 않고 containerd를 직접 사용
## OS 버전에 맞춰서 해당되는 패키지를 다운 받아 사용
## RHEL 7.x
wget https://download.docker.com/linux/centos/7/x86_64/stable/Packages/containerd.io-1.6.8-3.1.el7.x86_64.rpm
wget http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm
## RHEL 8.x
wget https://download.docker.com/linux/centos/8/x86_64/stable/Packages/containerd.io-1.6.9-3.1.el8.x86_64.rpm
wget https://rpmfind.net/linux/centos/8-stream/AppStream/ppc64le/os/Packages/container-selinux-2.195.1-1.module_el8.8.0+1254+78119b6e.noarch.rpm
sudo yum localinstall container*

# cri 활성화
# Kubernetes에서는 OCI 표준을 준수하는 이미지들을 실행할 수 있는 Container Runtime Interface, 이하 CRI 스펙을 버전 1.5부터
# 제공함으로써 Docker 버전과 무관하게 OCI 표준을 준수하기만 하면 어떤 컨테이너 이미지도 Kubernetes에서 실행가능한 환경이 만들어지게 되었습니다.
sudo sed -i 's/disabled_plugins/#disabled_plugins/g' /etc/containerd/config.toml
sudo systemctl enable containerd.service
sudo systemctl start containerd.service
 
cat << EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout : 3
EOF

# 패키지 설치
sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --cri-socket=unix:/var/run/containerd/containerd.sock --control-plane-endpoint=stg-k8s.devocean.co.kr --upload-certs
 
### 아래 실행결과 참고
[sudo_user@DEVOCEAN-K8SMASTER-STG01 ~]$ sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --cri-socket=unix:/var/run/containerd/containerd.sock --control-plane-endpoint=stg-k8s.devocean.co.kr --upload-certs
W1025 11:44:40.705158    8209 version.go:104] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": dial tcp 34.107.204.206:443: connect: connection timed out
W1025 11:44:40.705224    8209 version.go:105] falling back to the local client version: v1.25.3
[init] Using Kubernetes version: v1.25.3
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [devocean-k8smaster-stg01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local stg-k8s.devocean.co.kr] and IPs [10.96.0.1 172.18.243.xxx]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [devocean-k8smaster-stg01 localhost] and IPs [172.18.243.xxx 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [devocean-k8smaster-stg01 localhost] and IPs [172.18.243.xxx 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 5.002202 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
86506fd09df763c840b6e75aebf804bd544aa91ec2cbb4ed9da449e6fc59838a
[mark-control-plane] Marking the node devocean-k8smaster-stg01 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node devocean-k8smaster-stg01 as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: i2xtli.04p1v34662sblr3e
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy
 
Your Kubernetes control-plane has initialized successfully!
 
To start using your cluster, you need to run the following as a regular user:
 
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
Alternatively, if you are the root user, you can run:
 
  export KUBECONFIG=/etc/kubernetes/admin.conf
 
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
 
You can now join any number of the control-plane node running the following command on each as root:
 
  kubeadm join stg-k8s.devocean.co.kr:6443 --token i2xtli.04p1v34662sblr3e \
    --discovery-token-ca-cert-hash sha256:26822204f065a612ad837d912fb2485cdf9985fde77ab5ccee0b6583bdc71ae6 \
    --control-plane --certificate-key 86506fd09df763c840b6e75aebf804bd544aa91ec2cbb4ed9da449e6fc59838a
 
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
 
Then you can join any number of worker nodes by running the following on each as root:
 
kubeadm join stg-k8s.devocean.co.kr:6443 --token i2xtli.04p1v34662sblr3e \
    --discovery-token-ca-cert-hash sha256:26822204f065a612ad837d912fb2485cdf9985fde77ab5ccee0b6583bdc71ae6
 
# join 명령어 복사 필요
 
# sudo_user 계정에서 kubectl 명령어 사용을 위해 아래 작업 진행
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# CNI 설명 및 비교 참고
# https://ykarma1996.tistory.com/179
 
# Calico CNI플러그인이 가장 보편적이고 다양한 기능을 제공하면서 성능적으로도 매우 준수하고 레퍼런스가 가장 많기 때문에 Calico 적용
 
cd ~ && mkdir -p K8s_installs/calico && cd ~/K8s_installs/calico
wget https://raw.githubusercontent.com/projectcalico/calico/v3.24.3/manifests/tigera-operator.yaml
wget https://raw.githubusercontent.com/projectcalico/calico/v3.24.3/manifests/custom-resources.yaml
 
# custom-resources.yaml 에서 cidr 수정
# cidr : 10.244.0.0/16은 현재 on-premise 센터 이용 대역이 아니므로 사용가능
# 다른 대역은 네트워크 문제 발생 할 수 있음
# vxlan.calico 인터페이스를 새로 생성할때 eth0을 기본을 지정하는것으로 판단된
# 명시적을 nodeAddressAutodetectionV4: 지정 필요
# https://github.com/projectcalico/calico/issues/2561
 
아래 이미지 참고해서 nodeAddressAutodetectionV4 지시어 적용
 
kubectl create -f tigera-operator.yaml
kubectl create -f custom-resources.yaml

# 재기동 필요시
# kubectl rollout restart daemonset calico-node-????? calico-system

sudo kubeadm join stg-k8s.devocean.co.kr:6443 --token i2xtli.04p1v34662sblr3e \
    --discovery-token-ca-cert-hash sha256:26822204f065a612ad837d912fb2485cdf9985fde77ab5ccee0b6583bdc71ae6 \
    --control-plane --certificate-key 86506fd09df763c840b6e75aebf804bd544aa91ec2cbb4ed9da449e6fc59838a
 
# 토큰 만료시(2시간 만료)
# sudo kubeadm init phase upload-certs --upload-certs
# 아래 명령어로 토큰 확인
# sudo kubeadm token list
# 생성된 certificate-key 이용 join command 생성
# sudo kubeadm token create --print-join-command --certificate-key xxxxxxxxxxxxxxxxxxxx

export ETCD_VER=v3.5.4
curl https://storage.googleapis.com/etcd/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -O
tar xzvf etcd-${ETCD_VER}-linux-amd64.tar.gz
sudo mv etcd-${ETCD_VER}-linux-amd64/etcdctl /usr/local/bin/etcdctl
etcdctl version
 
rm -rf etcd-${ETCD_VER}-linux-amd64/
rm -f etcd-${ETCD_VER}-linux-amd64.tar.gz
 
# 실제 STG-K8s 상태 확인
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /home/sudo_user>$ sudo /usr/local/bin/etcdctl --write-out=table  --cert="/etc/kubernetes/pki/etcd/peer.crt" --key="/etc/kubernetes/pki/etcd/peer.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints https://devocean-k8smaster-stg01:2379 member list
+------------------+---------+-----------------------------+-----------------------------+-----------------------------+------------+
|        ID        | STATUS  |            NAME             |         PEER ADDRS          |        CLIENT ADDRS         | IS LEARNER |
+------------------+---------+-----------------------------+-----------------------------+-----------------------------+------------+
| 986a56cf5fa03ddc | started | devocean-k8smaster-stg02 | https://172.18.243.xxx:2380 | https://172.18.243.xxx:2379 |      false |
| 9898ba81150c173f | started | devocean-k8smaster-stg01 | https://172.18.243.xxx:2380 | https://172.18.243.xxx:2379 |      false |
| f59fa1dc09df26e4 | started | devocean-k8smaster-stg03 | https://172.18.243.xxx:2380 | https://172.18.243.xxx:2379 |      false |
+------------------+---------+-----------------------------+-----------------------------+-----------------------------+------------+
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /home/sudo_user>$ sudo /usr/local/bin/etcdctl --write-out=table  --cert="/etc/kubernetes/pki/etcd/peer.crt" --key="/etc/kubernetes/pki/etcd/peer.key" --cacert="/etc/kubernetes/pki/etcd/ca.crt" --endpoints https://devocean-k8smaster-stg01:2379,https://devocean-k8smaster-stg02:2379,https://devocean-k8smaster-stg03:2379 endpoint status
+------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|                 ENDPOINT                 |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://devocean-k8smaster-stg01:2379 | 9898ba81150c173f |   3.5.4 |  6.1 MB |      true |      false |         3 |      81668 |              81668 |        |
| https://devocean-k8smaster-stg02:2379 | 986a56cf5fa03ddc |   3.5.4 |  7.2 MB |     false |      false |         3 |      81668 |              81668 |        |
| https://devocean-k8smaster-stg03:2379 | f59fa1dc09df26e4 |   3.5.4 |  7.2 MB |     false |      false |         3 |      81668 |              81668 |        |
+------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+

sudo kubeadm join stg-k8s.devocean.co.kr:6443 --token p12mus.w4lcm1mjbrb3esro \
    --discovery-token-ca-cert-hash sha256:26822204f065a612ad837d912fb2485cdf9985fde77ab5ccee0b6583bdc71ae6
 
# 아래와 같이 토근 만료 여부 확인
$ kubeadm token list
TOKEN                     TTL         EXPIRES                USAGES                   DESCRIPTION                                                EXTRA GROUPS
p12mus.w4lcm1mjbrb3esro   19h         2022-11-01T08:56:55Z   authentication,signing   <none>                                                     system:bootstrappers:kubeadm:default-node-token
 
# discovery-token-ca-cert-hash를 잊어 버렸으면 아래 명령어로 확인
# openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
#토큰 만료시(2시간 만료) 아래 명령어로 토큰과 해시 확인
sudo kubeadm init phase upload-certs --upload-certs
# 위에서 출력된 Using certificate key 값을 이용 아래 명령어로 토큰 생성 및 조인 명령어 출력
# sudo kubeadm token create --certificate-key="3466bd23afce9ce1eb75dc5f2579aa2ba2c22b3179d75d243a48b816960f0957" --print-join-command
# 출력된 명령어중 --control-plane 옵션제거
 
# 운영 편의를 위해 label 추가하여
kubectl label node devocean-k8snode-stg01 node-role.kubernetes.io/worker=worker
kubectl label node devocean-k8snode-stg02 node-role.kubernetes.io/worker=worker
kubectl label node devocean-k8snode-stg03 node-role.kubernetes.io/worker=worker
kubectl label node devocean-k8slb-stg01 node-role.kubernetes.io/LB=LB
kubectl label node devocean-k8slb-stg02 node-role.kubernetes.io/LB=LB

# Onpremise 에서 동적으로 Load Balance 지원을 위해 LB 어플리케이션 이용
 
# taint 설정 - LB전용 노드에 metalLB 운영
# taint 설정을 스케쥴로가 해당 노드에 파드 할당을 안하다록 할 수 있음
# toleration 적용 및 조건에 맞다면 taint 설정이 되어 있더라도 할당
#https://gruuuuu.github.io/cloud/k8s-taint-toleration/ 참고
# 제거는 kubectl taint node devocean-k8slb-stg01 LB=MetalLB:NoSchedule-
kubectl taint node devocean-k8slb-stg01 LB=MetalLB:NoSchedule
kubectl taint node devocean-k8slb-stg02 LB=MetalLB:NoSchedule
 
# MetalLB yaml 다운받아 pod 구성
wget https://raw.githubusercontent.com/metallb/metallb/v0.13.7/config/manifests/metallb-native.yaml
# metallb-native.yaml 내용중 speaker 에 대해 LB 전용 서버에서 할당 하기 위해 아래 이미지 참고하여 수정 필요

# configMap 작성 아래 내용 참고
# https://metallb.universe.tf/configuration/
# 예제 https://github.com/metallb/metallb/blob/main/configsamples/deployment_l2.yaml
# MetalLB는 BGP와 L2레벨 지원이 되나 데이터센터 네트워크 환경상  BGP 사용이 어려워 L2 모드로 구성
 
# IP 자동 할당을 위해
cat << EOF | tee l2.yml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: private-vip
  namespace: metallb-system
spec:
  addresses:
  - 172.18.243.160-172.18.243.165
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: private-vip
  namespace: metallb-system
EOF
 
# IP 고정 할당을 위해
# ingress 전용 IP 2개 학보 목적
cat << EOF | tee l2-fixed.yml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: fixed-private-vip
  namespace: metallb-system
spec:
  addresses:
  - 172.18.243.166-172.18.243.167
  autoAssign: false
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: fixed-private-vip
  namespace: metallb-system
EOF
 
kubectl apply -f ./l2.yml
kubectl apply -f ./l2-vip.yml
 
>$ kubectl get IPAddressPool  -n metallb-system
NAME                AUTO ASSIGN   AVOID BUGGY IPS   ADDRESSES
fixed-private-vip   false         false             ["172.18.243.166-172.18.243.167"]
private-vip         true          false             ["172.18.243.160-172.18.243.165"]

# 특정IP 고정 또는 특정 풀에서 받고 싶다면 Service 생성시 annotations 이용 적용 할 수 있음
지정IP - metallb.universe.tf/loadBalancerIPs
지정Pool - metallb.universe.tf/address-pool

# 설치 메뉴얼 참고
# https://docs.netapp.com/us-en/trident-2201/trident-get-started/kubernetes-deploy-operator.html#deploy-the-trident-operator-by-using-helm
# K8s 환경 operator 설치 이후 진행 사항 https://docs.netapp.com/us-en/trident-2201/trident-get-started/kubernetes-postdeployment.html#step-1-create-a-backend
# trident 설치 -> kubectl을 통해 backend 구성 -> Storage class 생성-> PVC 
# 참고로 메뉴얼상 "CRD"는 "CustomResourceDefinition"을 말함
 
#Netapp 솔루션을 사용하고자 trient 이용
 
mkdir trident
wget https://github.com/NetApp/trident/releases/download/v22.10.0/trident-installer-22.10.0.tar.gz
tar -xf trident-installer-22.10.0.tar.gz
cd trident-installer
# K8s 1.16 이상일때 아래 post yaml이용 - custom resourece definition
kubectl create -f ./deploy/crds/trident.netapp.io_tridentorchestrators_crd_post1.16.yaml
 
# 네임스페이스 생성 - trident
kubectl apply -f deploy/namespace.yaml
 
# 참고로 네임스페이스를 변경하고 싶으면 serviceaccount.yaml, clusterrolebinding.yaml , operator.yaml 수정 필요
# update the YAML manifests and generate your bundle.yaml using the kustomization.yaml
# kustomiz를 통해 serviceaccount.yaml, clusterrolebinding.yaml , operator.yaml 통합
# kubectl kustomize deploy/ > deploy/bundle.yaml
# 생성
# kubectl create -f ./deploy/bundle.yaml
# 그러니 1.25 이상부터는 PodSecurityPolicy 가 제거 되어서 아래 제공하는 bundle 이용
# 아래 "Enhancements" 이미지 참고
#
# 추가로 nodeSelector를 Woker node에만 pod가 생성 되도록 아래와 같이 수정(운영편의)
#      nodeSelector:
#        node-role.kubernetes.io/worker: woker
 
kubectl create -f ./deploy/bundle_post_1_25.yaml
 
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /home/sudo_user/K8s_installs/trident/trident-installer>$ kubectl get pods -n trident
NAME                                READY   STATUS    RESTARTS   AGE
trident-operator-55c9b69c48-zlpql   1/1     Running   0          4m51s

# operator을 이용 trident 설치를 위해 TridentOrchestrator 생성
# https://docs.netapp.com/us-en/trident-2201/trident-get-started/kubernetes-customize-deploy.html
# controle plane과 LB는 제외하기 위해 내용 수정
kubectl create -f deploy/crds/tridentorchestrator_cr.yaml
# trident 설치 완료 확인
kubectl describe torc trident

#설치 확인
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /home/sudo_user/K8s_installs/trident/trident-installer>$./tridentctl -d -n trident version
Operating mode = tunnel, Trident pod = trident-csi-5995bcdc78-ljlmf, Namespace = trident, CLI = kubectl
Invoking tunneled command: kubectl exec trident-csi-5995bcdc78-ljlmf -n trident -c trident-main -- tridentctl version -o json
Version JSON: {
  "server": {
    "version": "22.10.0",
    "majorVersion": 22,
    "minorVersion": 10,
    "patchVersion": 0,
    "preRelease": "",
    "buildMetadata": "",
    "apiVersion": "1",
    "goVersion": "go1.18.7"
  },
  "client": {
    "version": "22.10.0",
    "majorVersion": 22,
    "minorVersion": 10,
    "patchVersion": 0,
    "preRelease": "",
    "buildMetadata": "",
    "apiVersion": "1",
    "goVersion": "go1.18.7"
  }
}
 
+----------------+----------------+
| SERVER VERSION | CLIENT VERSION |
+----------------+----------------+
| 22.10.0        | 22.10.0        |
+----------------+----------------+

# 이후 작업은 backend 설정 -> Storage Class 설정
# tridentctl 과 kubectl 둘다 이용 할 수 있게
# 아래는 kubectl을 이용해서 backend 설정 진행한 예시
# trident에서 제공하는 sample yaml 활용
# /home/sudo_user/K8s_installs/trident/trident-installer/sample-input/backends-samples/ontap-nas/backend-tbc-ontap-nas.yaml
# backend 설정시 필요한 managementLIF, dataLIF 등은 데이터센터 스토리지 담당자 협조 요청 필요
 
cat << EOF | tee ./backend-tbc-ontap-nas.yaml
apiVersion: v1
kind: Secret
metadata:
  name: backend-tbc-ontap-nas-secret
type: Opaque
stringData:
  username: trident_user
  password: test1234
---
apiVersion: trident.netapp.io/v1
kind: TridentBackendConfig
metadata:
  name: backend-tbc-ontap-nas
spec:
  version: 1
  storageDriverName: ontap-nas
  managementLIF: 10.40.xxx.xxx
  dataLIF: 192.168.xxx.xxx
  backendName: dpp-fas8300
  svm: svm_nas
  limitVolumeSize: 20Gi
  credentials:
    name: backend-tbc-ontap-nas-secret
EOF
# backend 생성
kubectl -n trident create -f backend-tbc-ontap-nas.yaml

# storage cloass 생성
 
cat << EOF | tee ./storage-class-ontapnas.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ontap-nas
provisioner: csi.trident.netapp.io
parameters:
  backendType: "ontap-nas"
  media: "hdd"
  provisioningType: "thin"
  snapshots: "true"
reclaimPolicy: Retain
EOF
 
kubectl create -f ./storage-class-ontapnas.yaml

# 기본적으로 default StorageClass 설정이 없기 떼문에 추가
kubectl patch storageclass ontap-nas -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
# delete는
# kubectl patch storageclass ontap-nas -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'

# 생성 테스트 진행
# pvc를 통한 pv 생성 및 pod 마운트 테스트
cat << EFO | tree ./task-pv-pod.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-default
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 20Gi
---
kind: Pod
apiVersion: v1
metadata:
  name: task-pv-pod
spec:
  volumes:
    - name: task-pv-storage
      persistentVolumeCl
       claimName: basic
  containers:
    - name: task-pv-container
      image: nginx
      ports:
        - containerPort: 80
          name: "http-server"
      volumeMounts:
        - mountPath: "/usr/share/nginx/html"
          name: task-pv-storage
EOF
kubectl create -f ./task-pv-pod.yaml

# yaml 파일 다운로드 및 LB 전용 node에서 운영 될 수 있도록 tolerations 적용
cd ~ && mkdir -p K8s_installs/ingress-nginx && cd ~/K8s_installs/ingress-nginx
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/cloud/deploy.yaml
# 172.18.243.167 고정 IP 할당 Loadbalance에 그리고 LB node에만 controller 할당

kubectl create -f deploy.yaml
 
>$ kubectl get pod ingress-nginx-controller-7b989bd858-bqprx -n ingress-nginx -o wide
NAME                                        READY   STATUS    RESTARTS   AGE   IP             NODE                      NOMINATED NODE   READINESS GATES
ingress-nginx-controller-7b989bd858-bqprx   1/1     Running   0          55s   10.244.208.1   devocean-k8slb-stg01   <none>           <none>

# pyenv 설치(python 최신 버전 활요하고자 함)
 
sudo yum install -y gcc gcc-c++ make git patch openssl-devel zlib-devel readline-devel sqlite-devel bzip2-devel xz-devel libffi-devel openldap-devel
git clone https://github.com/pyenv/pyenv.git ~/.pyenv
echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> .bashrc
echo 'eval "$(pyenv init --path)"' >> .bashrc
source ~/.bashrc
git clone https://github.com/pyenv/pyenv-virtualenv.git $(pyenv root)/plugins/pyenv-virtualenv
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
source ~/.bash_profile
sudo yum install openssl11-devel openssl11-lib
CPPFLAGS=$(pkg-config --cflags openssl11) LDFLAGS=$(pkg-config --libs openssl11) pyenv install -v 3.10.8
 
# flask 및 라이브러리 설치
pip install python-ldap flask requests
# nginx + gunicorn 조합으로 app 연동
pip install gunicorn

# flask
cat << EOF | tee ldap-webhook.py
from flask import Flask, request, jsonify
import pprint
import requests
import ldap
from requests.auth import HTTPBasicAuth
 
app = Flask(__name__)
 
@app.route('/', methods=['POST'])
 
def auth():
 
        # User가 kubernetes API Server에 인증 요청
        # Kubernetes API Server 가 사전에 정의 된 Webhook Server로 REST 요청보낸것을 받아오는 코드
        tokenReview = request.json
        print("\n")
        print("\n")
        pprint.pprint('---return result---')
        pprint.pprint(tokenReview)
 
        print("\n")
 
        # Webhook Server에 연동되어있는 인증 서버에서 인증 결과 받아옴
        tokenReview['status'] = external_auth_LDAP(tokenReview)
        pprint.pprint('---return result---')
        pprint.pprint(tokenReview)
 
 
        # Webhook Server에서 Kubernetes API Server로 인증결과 보냄
        return jsonify(tokenReview)
 
 
# 외부 인증 시스템
def external_auth_LDAP(tokenReview):
        try:
                user, pw = tokenReview['spec']['token'].split (':')
 
                # 예제에서 생성했던 도메인 정보 (/etc/hosts 에 localhost로 명시했음)
                ldap_address = "ldap://172.27.xxx.xxx:389"
 
                ldap_object = initialize_ldap(ldap_address)
                ldap_result = authenticate(ldap_object, ldap_address, user, pw)
 
                print('result: %s'%(ldap_result))
 
                if ldap_result == True:
                        status = {}
                        status['authenticated'] = True
                        status['user'] = {
                                'username': user,
                                'uid': user,
                                'groups': ['apollo']
                        }
                else :
                        status = {}
                        status['authenticated'] = False
        except:
                status = {}
                status['authenticated'] = False
        return status
 
 
def authenticate(ldap_object, ldap_address, user_name, password):
        try:
                ldap_object.simple_bind_s(user_name, password)
        except ldap.INVALID_CREDENTIALS:
                ldap_object.unbind()
                return False
        except Exception as e:
                print(e)
                return False
        return True
 
 
 
def initialize_ldap(ldap_address):
        ldap_object = ldap.initialize(ldap_address)
        return ldap_object
 
 
if __name__ == '__main__':
        app.run()
EOF
 
 
# gunicorn을 systemd에 등록
cat << EOF | sudo tee /usr/lib/systemd/system/ldap-webhook.service
[Unit]
Description=gunicorn daemon
After=network.target
 
[Service]
User=suser
Group=suser
WorkingDirectory=/skt/service/webhook
ExecStart=/home/suser/.pyenv/shims/gunicorn \
        --workers 4 \
        --bind unix:/skt/service/webhook/run/gunicorn.sock \
        ldap-webhook:app
 
[Install]
WantedBy=multi-user.target
EOF
 
sudo systemctl enable ldap-webhook
 
# Nginx 설치는 생략....

# Control plain node 전부 kube-api 적용 필요
cat << EOF | sudo tee -a /etc/kubernetes/pki/ldap-webhook.yaml
apiVersion: v1
kind: Config
clusters:
- name: ldap-webhook
cluster:
server: http://172.27.xxx.xxx:6000
insecure-skip-tls-verify: true
users:
- name: kube-apiserver
contexts:
- context:
cluster: ldap-webhook
user: kube-apiserver
name: ldap-webhook
current-context: ldap-webhook
EOF
 
# /etc/kubernetes/manifests/kube-apiserver.yaml 추가 수정
 
    - --authentication-token-webhook-config-file=/etc/kubernetes/pki/ldap-webhook.yaml
    - --authentication-token-webhook-cache-ttl=5m

# kube-apiserver 재기동 방법
kube-system에 기본적으로 설치되는 pod들은 전부 /etc/kubernetes/manifests 이하에 yaml 파일 존재 여부를 항상 모니터링 함
즉, 해당 yaml 파일이 존재 하지 않으면 pod가 제거되고 다시 yaml 파일이 생성되면 해당 내용으로 pod 기동함
 
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /etc/kubernetes/manifests>$ ll
total 16
-rw------- 1 root root 2424 Oct 25 11:44 etcd.yaml
-rw------- 1 root root 3490 Dec  7 21:40 kube-apiserver.yaml
-rw------- 1 root root 2854 Oct 25 11:44 kube-controller-manager.yaml
-rw------- 1 root root 1440 Oct 25 11:44 kube-scheduler.yaml
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /etc/kubernetes/manifests>$ mv kube-apiserver.yaml ../
# ps 명령어로 kube-apiserver 확인 후 사라졌으면 다시 복귀
[sudo_user@DEVOCEAN-K8SMASTER-STG01] /etc/kubernetes/manifests>$ mv ../kube-apiserver.yaml .
# 인증서 갱신도 동일하게 적용 하면 됨