daos-operator/hack/ci/node-prep.sh at main · gluesys/daos-operator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Gluesys Co., Ltd.
#
# 노드 안에서 실행. 사용: node-prep.sh <worker|cp|runner> <hostname> <ens19-ip> <ib-ip>
# 설계 3절 + testbed-k8s-exaci4-2.md 의 EL8 특이사항(conntrack, cgroupfs, crun).
set -euo pipefail
role=$1; name=$2; ip2=$3; ibip=$4
K8S_VER=1.31.14

hostnamectl set-hostname "$name"
# 기본 이미지에 DHCP 프로파일 ens19 가 이미 있을 수 있다(J 노드). 있든 없든 고정 주소로 고친다.
nmcli -t -f NAME con show | grep -qx ens19 || nmcli con add type ethernet ifname ens19 con-name ens19
nmcli con modify ens19 ipv4.method manual ipv4.addresses "$ip2/24" ipv6.method disabled connection.autoconnect yes
nmcli con up ens19 >/dev/null

swapoff -a; sed -i '/\sswap\s/d' /etc/fstab
setenforce 0 || true; sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config
systemctl disable --now firewalld || true

# THP never (즉시 + 영구)
echo never > /sys/kernel/mm/transparent_hugepage/enabled
grubby --update-kernel=ALL --args="transparent_hugepage=never"

cat > /etc/modules-load.d/k8s.conf <<M
overlay
br_netfilter
M
modprobe overlay; modprobe br_netfilter
cat > /etc/sysctl.d/99-k8s.conf <<S
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
S
sysctl --system >/dev/null

if [ "$role" = worker ]; then
    echo uio_pci_generic > /etc/modules-load.d/uio.conf; modprobe uio_pci_generic
    echo "vm.nr_hugepages = 4096" > /etc/sysctl.d/90-hugepages.conf
    sysctl -w vm.nr_hugepages=4096 >/dev/null
    # IB: 기본 이미지에 rdma-core 가 없어 mlx5_core 만 뜨고 netdev 가 없다. mlx5_ib + ib_ipoib 가
    # 있어야 IPoIB 주소와 파드의 /dev/infiniband(verbs) 가 생긴다. (jenkins_lib ExaCI.src 도 같은 사정)
    dnf install -y -q rdma-core libibverbs infiniband-diags
    printf 'mlx5_ib\nib_ipoib\n' > /etc/modules-load.d/ib.conf; modprobe mlx5_ib; modprobe ib_ipoib; sleep 3
    # IPoIB (verbs 프로파일용). VF 인터페이스 이름은 보장되지 않으므로 첫 ib* 를 쓴다. 없으면 건너뛴다.
    ibif=$(ls /sys/class/net | grep -m1 '^ib' || true)
    if [ -n "$ibif" ]; then
        nmcli -t -f NAME con show | grep -qx ibci || nmcli con add type infiniband ifname "$ibif" con-name ibci
        nmcli con modify ibci ipv4.method manual ipv4.addresses "$ibip/24" ipv6.method disabled connection.autoconnect yes
        nmcli con up ibci >/dev/null || true
    fi
fi

# 저장소: pkgs.k8s.io (kubeadm 1.31.x), CRI-O 1.31
cat > /etc/yum.repos.d/kubernetes.repo <<R
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
R
cat > /etc/yum.repos.d/cri-o.repo <<R
[cri-o]
name=CRI-O
baseurl=https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.31/rpm/repodata/repomd.xml.key
R
dnf install -y -q conntrack-tools iproute-tc socat crun chrony
dnf install -y -q --disableexcludes=kubernetes "kubelet-$K8S_VER" "kubeadm-$K8S_VER" "kubectl-$K8S_VER" cri-o
cat > /etc/crio/crio.conf.d/10-cgroupfs.conf <<C
[crio.runtime]
cgroup_manager = "cgroupfs"
conmon_cgroup = "pod"
default_runtime = "crun"
C
systemctl enable --now crio chronyd
systemctl enable kubelet
echo "node-prep $role $name: done ($(rpm -q cri-o))"