Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: ## base64로 코딩된 K8S CA 인증서
    server: https://k8s.foo.com:6443  ## kube-api Server의 주소와 포트
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
  user:
    client-certificate-data: ## base64로 코딩된 K8S CA가 발행 한 Client 인증서
    client-key-data: ## base64로 코딩된 Client 인증에 사용된 Private Key 

O=kubeadm:cluster-admins, CN=kubernetes-admin

$ kubectl describe clusterrolebinding kubeadm:cluster-admins
Name:         kubeadm:cluster-admins
Labels:       <none>
Annotations:  <none>
Role:
  Kind:  ClusterRole
  Name:  cluster-admin
Subjects:
  Kind   Name                    Namespace
  ----   ----                    ---------
  Group  kubeadm:cluster-admins

$ kubectl describe clusterrole cluster-admin
Name:         cluster-admin
Labels:       kubernetes.io/bootstrapping=rbac-defaults
Annotations:  rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
  Resources  Non-Resource URLs  Resource Names  Verbs
  ---------  -----------------  --------------  -----
  *.*        []                 []              [*]
             [*]                []              [*]

token,user,uid,"group1,group2,group3"

{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"*",         "nonResourcePath": "*", "readonly": true}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"admin",     "namespace": "*",              "resource": "*",         "apiGroup": "*"                   }}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:controller_manager", "namespace": "*",              "resource": "*"                   		}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kubelet",   "namespace": "*",              "resource": "*"                      		}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"kube_proxy",   "namespace": "*",              "resource": "*"}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:logging",   "namespace": "*",           "resource": "*"                  }}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:monitoring",   "namespace": "*",              "resource": "*"}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:serviceaccount:kube-system:default",   "namespace": "*", "resource": "*"}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"alice",     "namespace": "alice-space", "resource": "*",         "apiGroup": "*"                   }}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"bob",       "namespace": "alice-space", "resource": "*",         "apiGroup": "*", "readonly": true }}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"d6860ebb-b2f5-4eca-8ac6-3acd1cc17356",       "namespace": "shared-cluster-partition-test", "resource": "pods",         "apiGroup": "*"}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"d6860ebb-b2f5-4eca-8ac6-3acd1cc17356",       "namespace": "shared-cluster-partition-test", "resource": "namespace",         "apiGroup": "*", "readonly": true}}
{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"d6860ebb-b2f5-4eca-8ac6-3acd1cc17356",       "namespace": "shared-cluster-partition-test", "resource": "quota",         "apiGroup": "*", "readonly": true}}

## Admin Group을 위한 K8S RBAC 설정
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: devocean:admin   #admin 이란 이름이 일반적이라서 prefix를 붙였습니다.
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: devocean:admin #admin 이 일반적이라서 prefix 붙임 

## LeastPriviliges를 위한 K8S RBAC 설정
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: devocean:leastpriviliges
rules:    # 권한은 namespace와 node 자원에 대한 read
- apiGroups:
  - '*'
  resources:
  - namespaces
  - nodes
  verbs:
  - get
  - list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: devocean:leastpriviliges
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: devocean:leastpriviliges
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: devocean:leastpriviliges

## foo Group을 위한 K8S RBAC 설정
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: devocean:foo
  namespace: foo
rules:
- apiGroups:
  - '*'
  resources:
  - '*'
  verbs:
  - '*'
 ---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: devocean:foo
  namespace: foo
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: devocean:foo
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: devocean:foo

apiVersion: v1
kind: Service
metadata:
  name: keycloak
  labels:
    app: keycloak
spec:
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  - name: infinispan
    port: 7800
    targetPort: 7800
    protocol: TCP
  selector:
    app: keycloak
---
## headless Service
apiVersion: v1
kind: Service
metadata:
  name: infinispan
spec:
  clusterIP: None  ## headless Service 생성 방법
  selector:
    app: keycloak
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: keycloak
  labels:
    app: keycloak
spec:
  replicas: 2
  strategy:
    type: "RollingUpdate"
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
  selector:
    matchLabels:
      app: keycloak
  template:
    metadata:
      labels:
        app: keycloak
    spec:
      containers:
      - name: keycloak
        image: quay.io/keycloak/keycloak:21.0.1
        args: ["start"]
        env:
        - name: KC_HOSTNAME
          value: "keycloak.foo.com"   ## Global Access 가능한 DNS Name
        - name: KC_HTTP_RELATIVE_PATH
          value: "/auth"                         ## Admin console에 login하려면 설정 필요
        - name: KC_HTTP_ENABLED
          value: "true"
        - name: KC_HEALTH_ENABLED
          value: "true"
        - name: KC_DB
          value: "postgres"
        - name: KC_DB_PASSWORD
          value: "1q2w3e4r"
        - name: KC_DB_URL_HOST
          value: "postgre.foo.com"
        - name: KC_DB_URL_PORT
          value: "5432"
        - name: KC_DB_USERNAME
          value: "postgres"
        - name: KEYCLOAK_ADMIN
          value: "admin"
        - name: KEYCLOAK_ADMIN_PASSWORD
          value: "admin"
        - name: KC_PROXY
          value: "edge"                 ## reverse proxy 뒷단에 KC가 존재한다는 의미
        - name: KC_CACHE_STACK
          value: "kubernetes"
        - name: KC_TRANSACTION_XA_ENABLED       ## 2PC 지원여부. DB Clustering에 맞김
          value: "false"
        - name: JAVA_OPTS
          value: "-Djgroups.dns.query=infinispan.keycloak.svc.cluster.local"   ## TCP multicast를 위한 DNS 주소
        ports:
        - name: http
          containerPort: 8080
        - name: infinispan
          containerPort: 7800
        readinessProbe:
          httpGet:
            path: /auth/health
            port: 8080
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: keycloak
spec:
  ingressClassName: nginx  ## nginx ingress controller 가정
  rules:
  - host: keycloak.foo.com
    http:
      paths:
      - path: /auth
        pathType: Prefix
        backend:
          service:
            name: keycloak
            port:
              number: 8080
  tls:
  - hosts:
    - keycloak.foo.com
    secretName: keycloak-tls

kubectl get secret keycloak -n keycloak -o jsonpath={.data.admin-password} | base64 -d

참고
문 기고는 아래 REST API 기준으로 설멍합니다.
https://www.keycloak.org/docs-api/21.0.1/rest-api/index.html

export TOKEN=$(curl -k -X POST https://keycloak.foo.com/auth/realms/master/protocol/openid-connect/token -d grant_type=password -d username=admin -d password=password -d client_id=admin-cli | jq -r '.access_token')

curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d '{"realm":"org-a", "enabled":"true"}' https://keycloak.foo.com/auth/admin/realms

curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d '{"name" : "devocean:admin"}' https://keycloak.foo.com/auth/admin/realms/org-a/groups
curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d '{"name" : "devocean:leastpriviliges"}' https://keycloak.foo.com/auth/admin/realms/org-a/groups
curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d '{"name" : "devocean:foo"}' https://keycloak.foo.com/auth/admin/realms/org-a/groups

#boss.json
{
   "username": "boss",
   "enabled": "true",
   "credentials": [
     {
       "type": "password",
       "value": "qwer",
       "temporary": false ## 다음 로그인시, 비번 초기화 안됨
     }
   ]
}

#bob.json
{
   "username": "bob",
   "enabled": "true",
   "credentials": [
     {
       "type": "password",
       "value": "qwer",
       "temporary": false
     }
   ]
}

curl -v -k -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @boss.json  https://keycloak.foo.ocm/auth/admin/realms/org-a/users
curl -v -k -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @bob.json  https://keycloak.foo.com/auth/admin/realms/org-a/users

#get User ID
export BOSS=$(curl -v -k -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keyclaok.foo.com/auth/admin/realms/org-a/users?username\=boss | jq -r .[0].id)
export BOB=$(curl -v -k -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keyclaok.foo.com/auth/admin/realms/org-a/users?username\=bob | jq -r .[0].id)

#get Group ID
export K8SADMIN=$(curl -v -k -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keyclaok.foo.com/auth/admin/realms/org-a/groups?search\=devocean:admin | jq -r .[0].id)
export K8SUSER=$(curl -v -k -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keyclaok.foo.com/auth/admin/realms/org-a/groups?search\=devocean:priviliges | jq -r .[0].id)
export K8SFOO=$(curl -v -k -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keyclaok.foo.com/auth/admin/realms/org-a/groups?search\=devocean:foo | jq -r .[0].id)

# User 'boss'에 'decocean:admin' group 할당
curl -v -k -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keycloak.foo.com/auth/admin/realms/org-a/users/$BOSS/groups/$K8SADMIN

# User 'bob'에 'k8s:foo' group , 'k8s:user" group 할당
curl -v -k -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keycloak.foo.com/auth/admin/realms/org-a/users/$BOB/groups/$K8SUSER
curl -v -k -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN"  https://keycloak.foo.com/auth/admin/realms/org-a/users/$BOB/groups/$K8SFOO

##client-k8s.json
{
    "clientId" : "k8s",
    "enabled": true,
    "publicClient": true,
    "standardFlowEnabled": true,
    "directAccessGrantsEnabled":true, ## REST API를 통한 직접 접근. Token 검증등
    "protocol": "openid-connect"
}

curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @client.json https://keycloak.foo.com/auth/admin/realms/org-a/clients

#clientscope.json
{
   "name" : "oidc" ,
   "protocol" :  "openid-connect",
   "protocolMappers" : [
      {
        "name" : "groups" ,
        "protocol" : "openid-connect",
        "protocolMapper" : "oidc-group-membership-mapper",
        "consentRequired" : "false" ,
        "config": {
                     "full.path":"false",
                     "id.token.claim":"true",
                     "access.token.claim":"false",
                     "claim.name":"groups",
                     "userinfo.token.claim":"false"
                   }
      },
      {
        "name" : "user" ,
        "protocol" : "openid-connect",
        "protocolMapper" : "oidc-usermodel-property-mapper",
        "consentRequired": "false" ,
        "config": {
                     "user.attribute" : "username",
		    "full.path":"false",
                     "id.token.claim":"true",
                     "access.token.claim":"false",
                     "claim.name":"preferred_username",
                     "userinfo.token.claim":"false"
                   }
      }
   ]
}

curl -v -k POST -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @clientscope.json https://keycloak.foo.com/auth/admin/realms/org-a/client-scopes

#Get Client-Scope ID
export SCOPE=$(curl -v -k GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" https://keycloak.foo.com/auth/admin/realms/org-a/client-scopes | jq -r '.[]|select(.name=="oidc").id')

#Get Client ID
export CLIENT=$(curl -v -k GET -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" https://keycloak.foo.com/auth/admin/realms/org-a/clients?clientId\=k8s | jq -r .[0].id)

#client scope oidc를 k8s client의 default scope으로 등록하기
curl -v -k -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" https://keycloak.foo.com/auth/admin/realms/org-a/clients/$CLIENT/default-client-scopes/$SCOPE

curl -k -X POST  https://keycloak.foo.com/auth/realms/org-a/protocol/openid-connect/token -d grant_type=password -d username=bob -d password=qwer -d client_id=k8s -d scope=openid | jq -r .id_token

## /etc/kubernetes/manifests/kube-apiserver.yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
  ..
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
 ... ## OIDC 설정
    - --oidc-issuer-url=https://keycloak.foo.com/auth/realms/org-a
    - --oidc-client-id=k8s
    - --oidc-username-claim=preferred_username
    - --oidc-groups-claim=groups
 ...

kubectl oidc-login get-token --oidc-issuer-url=https://keycloak.foo.com/auth/realms/org-a --oidc-client-id=k8s --username=bob --password=qwer

apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: [당신은 K8S CA]
    server: https://[당신의 Kube API Server Endpoint. eg, boo.foo.com:6443]
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: oidc
    name: oidc-user@kubernetes
    current-context: oidc-user@kubernetes
kind: Config
preferences: {}
users:
- name: oidc
  user:
    exec:   ## User의 인증을 아래 설정에 따라 받아 온다는 설정입니다.
      apiVersion: client.authentication.k8s.io/v1beta1
      command: kubectl
      args:
      - oidc-login
      - get-token
      - --oidc-issuer-url=https://keycloak.foo.com/auth/realms/org-a
      - --oidc-client-id=k8s
      - --grant-type=password     

$ kubectl get ns
Username: bob
Password:
NAME               STATUS   AGE
calico-apiserver   Active   18d
calico-system      Active   18d
default            Active   18d
foo                Active   14d
kube-node-lease    Active   18d
kube-public        Active   18d
kube-system        Active   18d
tigera-operator    Active   18d

$ kubectl get pod -n kube-system
Error from server (Forbidden): pods is forbidden: User "https://keycloak.foo.com/auth/realms/org-a#bob" cannot list resource "pods" in API group "" in the namespace "kube-system"

$ cd
$ rm .kube/cache/oidc-login/*

$ kubectl get pod -n kube-system
Username: boss
Password:
NAME                                        READY   STATUS    RESTARTS         AGE
coredns-76f75df574-kqwnj                    1/1     Running   9 (4h21m ago)    18d
coredns-76f75df574-r9ng7                    1/1     Running   9 (4h21m ago)    18d
etcd-ip-172-31-105-199                      1/1     Running   9 (4h21m ago)    18d
kube-apiserver-ip-172-31-105-199            1/1     Running   4 (4h21m ago)    11d
kube-controller-manager-ip-172-31-105-199   1/1     Running   10 (4h21m ago)   18d
kube-proxy-cmr9z                            1/1     Running   9 (4h21m ago)    18d
kube-scheduler-ip-172-31-105-199            1/1     Running   10 (4h21m ago)   18d