쿠버네티스 노드 구성 및 관리

Node 구성 및 관리

Cordon(경계선)

Drain

nodeName

nodeSelector(Lable 사용)

Node affinity

Anti affinity

Taints(자물쇠)와 Tolerations(열쇠)

Node 구성 및 관리

Cordon(경계선)

•

배포가 되지 않도록 경계선을 치는 방법이다.

# 1. Cordon 실행
[root@m-k8s 6.2]# k get no
NAME     STATUS   ROLES                  AGE   VERSION
m-k8s    Ready    control-plane,master   20d   v1.22.0
w1-k8s   Ready    <none>                 20d   v1.22.0
w2-k8s   Ready    <none>                 20d   v1.22.0
w3-k8s   Ready    <none>                 20d   v1.22.0

[root@m-k8s 6.2]# k cordon w3-k8s
node/w3-k8s cordoned

[root@m-k8s 6.2]# k get no
NAME     STATUS                     ROLES                  AGE   VERSION
m-k8s    Ready                      control-plane,master   20d   v1.22.0
w1-k8s   Ready                      <none>                 20d   v1.22.0
w2-k8s   Ready                      <none>                 20d   v1.22.0
w3-k8s   Ready,SchedulingDisabled   <none>                 20d   v1.22.0

# 2. 배포 후 확인(w3-k8s에는 배포가 되지 않음)
[root@m-k8s 6.2]# k get po -o wide
NAME                            READY   STATUS              RESTARTS   AGE   IP               NODE     NOMINATED NODE   READINESS GATES
deploy-drain-5f6c9db768-cmkh7   1/1     Running             0          6s    172.16.103.168   w2-k8s   <none>           <none>
deploy-drain-5f6c9db768-qx64g   1/1     Running             0          6s    172.16.221.156   w1-k8s   <none>           <none>
deploy-drain-5f6c9db768-tnn8k   0/1     ContainerCreating   0          6s    <none>           w2-k8s   <none>           <none>
net                             1/1     Running             0          9d    172.16.132.23    w3-k8s   <none>           <none>
YAML
복사

Drain

•

Pod를 다른 Node로 이전하는 방법이다.

•

저절로 Cordon이 걸린다.

# 1. Deployment를 배포 후 확인
[root@m-k8s 6.2]# k get po -o wide
NAME                            READY   STATUS              RESTARTS   AGE   IP              NODE     NOMINATED NODE   READINESS GATES
deploy-drain-5f6c9db768-9qqgm   0/1     ContainerCreating   0          4s    <none>          w1-k8s   <none>           <none>
deploy-drain-5f6c9db768-jsjv9   0/1     ContainerCreating   0          4s    <none>          w2-k8s   <none>           <none>
deploy-drain-5f6c9db768-rr2jd   0/1     ContainerCreating   0          4s    <none>          w3-k8s   <none>           <none>
net                             1/1     Running             0          9d    172.16.132.23   w3-k8s   <none>           <none>

# 2. drain을 설정(1. Daemonset이 있고, 2. 단순 pod가 있어서 옵션을 줌)
[root@m-k8s 6.2]# k drain w3-k8s --ignore-daemonsets --force
node/w3-k8s already cordoned
WARNING: deleting Pods not managed by ReplicationController, ReplicaSet, Job, DaemonSet or StatefulSet: default/net; ignoring DaemonSet-managed Pods: kube-system/calico-node-lqtx2, kube-system/kube-proxy-rspm6, metallb-system/speaker-plhlg
evicting pod default/deploy-drain-5f6c9db768-rr2jd
evicting pod ingress-nginx/ingress-nginx-admission-patch-vkkn7
evicting pod default/net
evicting pod ingress-nginx/ingress-nginx-admission-create-r9rlz
pod/ingress-nginx-admission-patch-vkkn7 evicted
pod/ingress-nginx-admission-create-r9rlz evicted
pod/deploy-drain-5f6c9db768-rr2jd evicted
pod/net evicted
node/w3-k8s evicted

# w3에 있던 pod가 옮겨졌으며, Daemonset은 옮겨지지 않으니 삭제, 단순 pod는 삭제되었다.
[root@m-k8s 6.2]# k get po -o wide
NAME                            READY   STATUS    RESTARTS   AGE    IP               NODE     NOMINATED NODE   READINESS GATES
deploy-drain-5f6c9db768-46crn   1/1     Running   0          76s    172.16.103.171   w2-k8s   <none>           <none>
deploy-drain-5f6c9db768-9qqgm   1/1     Running   0          3m5s   172.16.221.157   w1-k8s   <none>           <none>
deploy-drain-5f6c9db768-jsjv9   1/1     Running   0          3m5s   172.16.103.170   w2-k8s   <none>           <none>
YAML
복사

nodeName

apiVersion: v1
kind: Pod
metadata:
  name: nodename
spec:
  containers:
  - name: nginx
    image: nginx
  nodeName: w3-k8s
YAML
복사

nodeSelector(Lable 사용)

•

노드의 Lable로 Pod를 배포한다.

# 1. Label 달기
[root@m-k8s 6.3]# k label node w3-k8s input=test
node/w3-k8s labeled

# 2. Label 확인하기
[root@m-k8s 6.3]# k get node --show-labels
NAME     STATUS   ROLES                  AGE   VERSION   LABELS
m-k8s    Ready    control-plane,master   21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=m-k8s,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=,node.kubernetes.io/exclude-from-external-load-balancers=
w1-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=w1-k8s,kubernetes.io/os=linux
w2-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=w2-k8s,kubernetes.io/os=linux
w3-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,input=test,kubernetes.io/arch=amd64,kubernetes.io/hostname=w3-k8s,kubernetes.io/os=linux

# 3. Label로 검색하기
[root@m-k8s 6.3]# k get node -l input=test
NAME     STATUS   ROLES    AGE   VERSION
w3-k8s   Ready    <none>   21d   v1.22.0

# 4. Label 제거하기
[root@m-k8s 6.3]# k label node w3-k8s input-
node/w3-k8s labeled

[root@m-k8s 6.3]# k get node --show-labels
NAME     STATUS   ROLES                  AGE   VERSION   LABELS
m-k8s    Ready    control-plane,master   21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=m-k8s,kubernetes.io/os=linux,node-role.kubernetes.io/control-plane=,node-role.kubernetes.io/master=,node.kubernetes.io/exclude-from-external-load-balancers=
w1-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=w1-k8s,kubernetes.io/os=linux
w2-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=w2-k8s,kubernetes.io/os=linux
w3-k8s   Ready    <none>                 21d   v1.22.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=w3-k8s,kubernetes.io/os=linux
YAML
복사

kubectl label node w1-k8s gpupool=nvidia accelerator=tesla-a100
kubectl label node w2-k8s gpupool=nvidia accelerator=tesla-v100
kubectl label node w3-k8s diskint=nvme inmemory=redis
----------------------------------------------------------------
# 노드 그룹이 되어 배포 시 노드 그룹 내 로드밸런싱 됨
apiVersion: v1
kind: Pod
metadata:
  name: nodeselector-gpupool
spec:
  containers:
  - name: nginx
    image: nginx
  nodeSelector:
    gpupool: nvidia
YAML
복사

Node affinity

•

NodeSelector보다 Node로의 배포조건을 다양하게 부여할 수 있다.

[배포 조건]

required DuringScheduling Ignored DuringExecution

preferred DuringScheduling Ignored DuringExecution

required DuringScheduling Required DuringExecution > 나중에 지원

preferred DuringScheduling Required DuringExecution > 나중에 지원

[연산자]

In NotIn > 키 값을 체크

Exists DoesNotExist > 키의 존재 여부 체크

Gt Lt > 키 값이 크고 작음을 체크

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nodeaffinity-preferred
  name: nodeaffinity-preferred
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nodeaffinity-preferred
  template:
    metadata:
      labels:
        app: nodeaffinity-preferred
    spec:
      containers:
      - image: nginx
        name: nginx
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: gpupool
                operator: In
                values:
                - nvidia
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: accelerator
                operator: In
                values:
                - tesla-a100
YAML
복사

Anti affinity

•

Node affinity의 조건을 반대로 하여 부여할 수 있다.

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: anti-nodeaffinity
  name: anti-nodeaffinity
spec:
  replicas: 3
  selector:
    matchLabels:
      app: anti-nodeaffinity
  template:
    metadata:
      labels:
        app: anti-nodeaffinity
    spec:
      containers:
      - image: nginx
        name: nginx
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: gpupool
                operator: In
                values:
                - nvidia
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 1
            preference:
              matchExpressions:
              - key: accelerator
                operator: NotIn
                values:
                - tesla-a100
YAML
복사

Taints(자물쇠)와 Tolerations(열쇠)

•

노드를 보호하는 방법이다.

[테인트와 톨러레이션 4가지 조건]

1. Effect

NoSchedule

•

가장 기본적인 설정. 노드에 테인트가 설정되어 있지 않은 경우 파드가 노드에 스케줄되지 않음. 이 경우 톨러레이션을 통한 배포만 가능함.

PreferNoschedule

•

NoSchedule과 유사하지만 스케줄러에서 더 이상 할당할 수 있는 노드가 없는 경우 테인트 설정을 무시하고 스케줄함

NoExecute

•

NoSchedule에 현재 할당된 파드에도 바로 적용되도록 스케줄을 다시 조정하는 기능 추가. 즉, 톨러레이션이 없는 파드는 모두 노드에서 제거함

2. Key : 사용자가 지정하는 기준 대상 키

3. Value : 사용자가 지정한 키에 대한 값

4. Operator : 키와 값에 대한 연산자(Exists | Equal) 기본 값은 Equal

[왜 마스터노드에 Pod가 Deploy 되지 않을까?]

→ 마스터노드에 Taints가 걸려져있기 때문

[마스터 노드에도 데몬셋 배포하기]

# 1. 마스터 노드에 걸려있는 Taints 설정 확인
[root@m-k8s 6.7]# k get node m-k8s -o yaml | grep -i taint -F5
  uid: 0e725559-d6a4-42a5-ac61-df62411cf3d7
spec:
  podCIDR: 172.16.0.0/24
  podCIDRs:
  - 172.16.0.0/24
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
status:
  addresses:
  - address: 192.168.1.10

#2. tolerations 데몬셋 배포하기
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: daemonset-w-tolerations
  labels:
    app: daemonset-w-tolerations
spec:
  selector:
    matchLabels:
      app: daemonset-w-tolerations
  template:
    metadata:
      labels:
        app: daemonset-w-tolerations
    spec:
      containers:
      - name: nginx
        image: nginx
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master

# 3. 마스터노드에 배포되었는지 확인
[root@m-k8s 6.7]# k get pod -o wide
NAME                            READY   STATUS              RESTARTS   AGE   IP               NODE     NOMINATED NODE   READINESS GATES
daemonset-w-tolerations-2rqzx   1/1     Running             0          6s    172.16.103.173   w2-k8s   <none>           <none>
daemonset-w-tolerations-b2cvt   0/1     ContainerCreating   0          7s    <none>           m-k8s    <none>           <none>
daemonset-w-tolerations-gx6l8   1/1     Running             0          7s    172.16.221.159   w1-k8s   <none>           <none>
daemonset-w-tolerations-vs2zg   0/1     ContainerCreating   0          7s    <none>           w3-k8s   <none>           <none>
net                             0/1     ImagePullBackOff    0          46h   172.16.221.158   w1-k8s   <none>           <none>
YAML
복사

[Taints로 DB 보호해보기]

# 1. 워커노드 3에 taint 걸기
[root@m-k8s 6.7]# k taint node w3-k8s DB=customer-info:NoSchedule
node/w3-k8s tainted

[root@m-k8s 6.7]# k get node w3-k8s -o yaml | grep -i taint -F5
  uid: 4476a393-3f08-43bd-a7db-93d5ee4ffb33
spec:
  podCIDR: 172.16.3.0/24
  podCIDRs:
  - 172.16.3.0/24
  taints:
  - effect: NoSchedule
    key: DB
    value: customer-info
status:
  addresses:

# 2. tolerations, affinty 설정 Deployment 배포하기
# (지정된 노드에만 tolerations 설정으로 배포)
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: deploy-w-tolerations-nodeaffinity
  name: deploy-w-tolerations-nodeaffinity
spec:
  replicas: 6
  selector:
    matchLabels:
      app: deploy-w-tolerations-nodeaffinity
  template:
    metadata:
      labels:
        app: deploy-w-tolerations-nodeaffinity
    spec:
      containers:
      - image: nginx
        name: nginx
      tolerations:
      - effect: NoSchedule
        key: DB
        value: customer-info
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: inmemory
                operator: In
                values:
                - redis

             0/1     ImagePullBackOff   0          46h     172.16.221.158   w1-k8s   <none>

[root@m-k8s 6.7]# k get pod -o wide
NAME                                                READY   STATUS              RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
deploy-w-tolerations-nodeaffinity-cb8474cb7-9vlrz   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
deploy-w-tolerations-nodeaffinity-cb8474cb7-ccq6g   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
deploy-w-tolerations-nodeaffinity-cb8474cb7-cqfj8   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
deploy-w-tolerations-nodeaffinity-cb8474cb7-fpd7t   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
deploy-w-tolerations-nodeaffinity-cb8474cb7-jttmv   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
deploy-w-tolerations-nodeaffinity-cb8474cb7-wb6vr   0/1     ContainerCreating   0          3s      <none>           w3-k8s   <none>           <none>
net                                                 0/1     ImagePullBackOff    0          46h     172.16.221.158   w1-k8s   <none>           <none>
YAML
복사