Собственный дистрибутив Kubernetes с нуля / Хабр

Содержание

k8s-release — инструмент сборки
Связь проектов
Как использовать
HA-кластер на Firecracker
Создай свой Talos или CoreOS Linux I

k8s-release — инструмент сборки

k8s-release — инструмент для сборки воспроизводимых DEB/RPM-пакетов компонентов Kubernetes (kubelet, kube-apiserver, etcd, Flannel, Istio и др.) в герметичном Docker-окружении. Создаёт подписанные apt/yum-репозитории, SBOM, хеш-суммы и airgap-бандлы для закрытых, регулируемых и изолированных сред. В репозитории также лежит Terraform-модуль для развёртывания HA Kubernetes на микро-ВМ Firecracker: 3 control-plane, 1+ workers, Flannel, Istio, SELinux enforcing.

k8s-release собирает любые комбинации компонентов через флаг --component. Можно указать отдельные компоненты (kubelet, etcd, flannel, istio), all для всего сразу, и формат пакетов — deb, rpm или оба.

./k8s-release build v1.36.1 --component kubelet,kubectl --format deb,rpm
./k8s-release build v1.36.1 --component all --format deb
./k8s-release build 3.5.22 --component etcd --format rpm
./k8s-release build 1.30.0 --component istio --format deb,rpm
./k8s-release bundle v1.36.1 --airgap
./k8s-release verify-release v1.36.1 --repo ingresslabs/k8s-release
./k8s-release passport v1.36.1

Связь проектов

Оба проекта — части одной экосистемы. Первый проект, k8s-release, использует ту же платформу для сборки и развёртывания Kubernetes на микро-ВМ Firecracker. Terraform-модуль в k8s-release автоматизирует развёртывание HA-кластера по тем же принципам, что описаны в статье. Вторая статья описывает самодельную Firecracker CI-платформу (загрузка за 10 мс, кеширование, снапшоты).

Как использовать

Подготовка:

git clone https://github.com/ingresslabs/k8s-release.git
cd k8s-release/terraform/k2vm-ha-lab
cp terraform.tfvars.example terraform.tfvars

Настройка хоста (один из двух):

echo 'K8S_RELEASE_PROOF_HOST=root@192.168.1.100' > ../../.env

Или target_host = "root@192.168.1.100" в terraform.tfvars.

На удалённом хосте должны быть: Firecracker, Docker, ядро, initrd, модули и подготовленный rootfs.ext4.

Запуск:

terraform init && terraform apply
terraform destroy

terraform {
  required_version = ">= 1.5.0"
  required_providers {
    local = { source = "hashicorp/local", version = "~> 2.5" }
    null  = { source = "hashicorp/null", version = "~> 3.2" }
  }
}

variable "name"              { type = string; default = "k8s-ha-lab" }
variable "target_host"       { type = string; default = "" }
variable "target_user"       { type = string; default = "root" }
variable "github_repo"       { type = string; default = "ingresslabs/k8s-release" }
variable "github_run_id"     { type = number; default = 26680027183 }
variable "control_plane_count" { type = number; default = 3 }
variable "worker_count"      { type = number; default = 2 }
variable "kubernetes_version" { type = string; default = "v1.36.1" }
variable "subnet_prefix"     { type = string; default = "198.19.2" }
variable "network_plugin"    { type = string; default = "flannel" }
variable "firecracker_binary" { type = string; default = "/usr/local/bin/firecracker" }
variable "kernel_path"       { type = string; default = "/opt/fc-lab/vmlinux" }
variable "initrd_path"       { type = string; default = "/opt/fc-lab/initrd.img" }
variable "kernel_modules_tar_path" { type = string; default = "/opt/fc-lab/modules.tar.gz" }
variable "base_rootfs_path"  { type = string; default = "/opt/fc-lab/rootfs.ext4" }
variable "vcpu_count"        { type = number; default = 2 }
variable "guest_selinux_mode" { type = string; default = "permissive" }
variable "enable_istio"      { type = bool;   default = false }
variable "redeploy_token"    { type = string; default = "" }

locals {
  repo_root   = abspath("${path.module}")
  env_path    = "${local.repo_root}/.env"
  env_content = fileexists(local.env_path) ? file(local.env_path) : ""
  env_lines   = [for l in split("\n", local.env_content) : trimspace(l)
    if l != "" && !startswith(l, "#")]
  env_pairs   = [for l in local.env_lines :
    regex("^([A-Za-z_][A-Za-z0-9_]*)=(.*)$", l)]
  env_map = { for p in local.env_pairs :
    p[0] => trimsuffix(trimprefix(p[1], "\""), "\"") }

  target_source = var.target_host != ""
    ? var.target_host
    : lookup(local.env_map, "K8S_RELEASE_PROOF_HOST", "")
  parts = can(regex("^([^@]+)@(.+)$", local.target_source))
    ? regex("^([^@]+)@(.+)$", local.target_source) : []
  effective_user = length(local.parts) == 2
    ? local.parts[0] : var.target_user
  effective_host = length(local.parts) == 2
    ? local.parts[1] : local.target_source

  selinux_params = var.guest_selinux_mode == "disabled"
    ? ["apparmor=0", "selinux=0", "audit=0"]
    : var.guest_selinux_mode == "permissive"
    ? ["apparmor=0", "security=selinux", "selinux=1", "enforcing=0"]
    : ["apparmor=0", "security=selinux", "selinux=1", "enforcing=1"]

  kernel_boot = join(" ", concat(
    ["console=ttyS0", "reboot=k", "panic=1", "pci=off",
     "root=/dev/vda", "rw"], local.selinux_params))

  manifest = {
    schema_version = "k2vm.spec.v1"
    name = var.name
    target = { host = local.effective_host,
      user = local.effective_user, workdir = "/root/${var.name}" }
    cluster = {
      distribution = "kubeadm"
      subnet_prefix = var.subnet_prefix
      control_plane_count = var.control_plane_count
      worker_count = var.worker_count
      network_plugin = var.network_plugin
      kubernetes_version = var.kubernetes_version
      control_plane_runtime = "static-pods"
    }
    firecracker = {
      binary = var.firecracker_binary
      bridge_name = "k2vmbr0"
      tap_prefix = "k2vmtap"
      kernel_source = "provided"
      kernel_path = var.kernel_path
      initrd_path = var.initrd_path
      kernel_modules_tar_path = var.kernel_modules_tar_path
      base_rootfs_path = var.base_rootfs_path
      kernel_params = local.kernel_boot
      vcpu_count = var.vcpu_count
    }
    guest = { selinux_mode = var.guest_selinux_mode }
    paths = { run_root = "/var/lib/k2vm-lab",
      cache_root = "/var/cache/k2vm-lab" }
    release = {
      enabled = true
      github_repo = var.github_repo
      github_run = { repo = var.github_repo,
        run_id = var.github_run_id }
      package_repository = {
        source = "github_run_artifact"
        artifact_layout = "component_packages"
        artifact_components = ["kubelet","kubectl",
          "kube-apiserver","kube-controller-manager",
          "kube-scheduler","kube-proxy","etcd","flannel"]
        mode = "hybrid"
      }
    }
    addons = { istio = { enabled = var.enable_istio,
      profile = "minimal" } }
    logging = { level = "INFO", format = "text" }
  }
}

resource "local_sensitive_file" "manifest" {
  filename = "${path.module}/${var.name}.rendered.json"
  content  = jsonencode(local.manifest)
}

resource "null_resource" "deploy" {
  triggers = {
    sha = sha256(local_sensitive_file.manifest.content)
    token = var.redeploy_token
  }
  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    command = <<-EOT
      set -e
      MANIFEST="${local_sensitive_file.manifest.filename}"
      HOST="${local.effective_user}@${local.effective_host}"
      scp "$MANIFEST" "$HOST:/tmp/lab-manifest.json"
      ssh $HOST "bash /opt/k2vm/k2vm-kubeadm-engine.sh apply /tmp/lab-manifest.json"
    EOT
  }
  provisioner "local-exec" {
    when = destroy
    interpreter = ["/bin/bash", "-lc"]
    command = <<-EOT
      ssh "${local.effective_user}@${local.effective_host}" \
        "bash /opt/k2vm/k2vm-kubeadm-engine.sh delete || true"
    EOT
  }
}

output "manifest_path" {
  value = local_sensitive_file.manifest.filename
}
output "target_host" { value = local.effective_host }

HA-кластер на Firecracker

Это руководство выполняется непосредственно на хосте Linux. Оно собирает одну переиспользуемую гостевую rootfs, клонирует её на несколько дисков микро-ВМ Firecracker, загружает:

3 узла control-plane kubeadm
встроенный etcd, по одному участнику на узел control-plane
2 рабочих узла по умолчанию
Kubernetes API через хост-прокси HAProxy

Предварительные требования

Выполняйте всё от root на хосте Linux, где будет запущен Firecracker. На хосте необходимы:

Docker
Firecracker
curl, ip, iptables, ssh, scp, ssh-keygen
unsquashfs из squashfs-tools
mkfs.ext4, e2fsck, resize2fs из e2fsprogs
mount, umount, truncate, chroot
исходящий доступ в интернет для: загрузки ядра LinuxKit, Ubuntu rootfs, пакетов Kubernetes, Docker containerd, манифеста Flannel, образа HAProxy

Выберите подсеть, не пересекающуюся с другими мостами на хосте.

Экспорт переменных

Создаёт 3 control-plane и 2 worker-узла по умолчанию. Измените WORKER_COUNT при необходимости.

export RUN_ROOT=/var/lib/firecracker-kubeadm-ha
export CACHE_ROOT=/var/cache/firecracker-kubeadm-ha
export CONTROL_PLANE_COUNT=3
export WORKER_COUNT=2
export NODE_COUNT="$((CONTROL_PLANE_COUNT + WORKER_COUNT))"
export SUBNET_PREFIX=198.19.0
export GATEWAY="${SUBNET_PREFIX}.1"
export CIDR="${SUBNET_PREFIX}.0/24"
export API_LB_IP="${SUBNET_PREFIX}.5"
export API_LB_PORT=6443
export PRIMARY_CONTROL_PLANE_IP="${SUBNET_PREFIX}.10"
export BRIDGE_NAME=k8sha198
export TAP_PREFIX=k8sha198
export FIRECRACKER_BIN=/usr/local/bin/firecracker
export FIRECRACKER_ARCH="$(uname -m)"
export LINUXKIT_KERNEL_IMAGE=linuxkit/kernel:6.12.59
export ROOTFS_SIZE_GIB=12
export CONTROL_PLANE_MEM_MIB=2048
export WORKER_MEM_MIB=1536
export VCPU_COUNT=2
export KUBERNETES_MINOR=v1.36
export KUBERNETES_VERSION=v1.36.1
export POD_CIDR=10.244.0.0/16
export SERVICE_CIDR=10.96.0.0/12
export CNI_PLUGINS_VERSION=v1.3.0
export NETWORK_PLUGIN=flannel
export FLANNEL_MANIFEST_URL="https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml"
export HAPROXY_IMAGE=haproxy:3.2.19-alpine
export API_LB_CONTAINER_NAME="kubeadm-ha-api-lb-${BRIDGE_NAME}"
export GUEST_SSH_KEY="${CACHE_ROOT}/lab_ssh_key"
export GUEST_SSH_PUB="${GUEST_SSH_KEY}.pub"
export ARTIFACT_PREFIX=/var/log/kubeadm-ha-lab
export KERNEL_BOOT_ARGS="console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw random.trust_cpu=on systemd.mask=serial-getty@ttyS0.service systemd.mask=systemd-random-seed.service"

Зависимости и SSH-ключ

Проверка зависимостей хоста и создание гостевого SSH-ключа.

command -v awk chroot curl docker e2fsck grep ip iptables mkfs.ext4 mount \
  resize2fs sha256sum sort ssh scp ssh-keygen truncate umount unsquashfs >/dev/null
test -x "${FIRECRACKER_BIN}"
mkdir -p "${RUN_ROOT}" "${CACHE_ROOT}"
if [[ ! -f "${GUEST_SSH_KEY}" || ! -f "${GUEST_SSH_PUB}" ]]; then
  ssh-keygen -q -t ed25519 -N "" -f "${GUEST_SSH_KEY}"
fi

Вспомогательные функции

node_role() {
  if (( $1 < CONTROL_PLANE_COUNT )); then
    echo control-plane
  else
    echo worker
  fi
}
node_ip() {
  printf '%s.%d\n' "${SUBNET_PREFIX}" "$((10 + $1))"
}
node_name() {
  printf 'k8s-%02d\n' "$1"
}
node_tap() {
  printf '%s%d\n' "${TAP_PREFIX}" "$1"
}
node_mac() {
  printf '06:36:00:00:00:%02x\n' "$((16 + $1))"
}
node_mem() {
  if [[ "$(node_role "$1")" == "control-plane" ]]; then
    echo "${CONTROL_PLANE_MEM_MIB}"
  else
    echo "${WORKER_MEM_MIB}"
  fi
}
SSH_OPTS=(-i "${GUEST_SSH_KEY}" -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5)
node_ssh() {
  local idx="$1"
  shift
  ssh "${SSH_OPTS[@]}" "root@$(node_ip "${idx}")" "$@"
}
server_ssh() {
  node_ssh 0 "$@"
}
copy_guest_file() {
  local remote_path="$1"
  local local_path="$2"
  if server_ssh "test -f $(printf '%q' "${remote_path}")" >/dev/null 2>&1; then
    server_ssh "cat $(printf '%q' "${remote_path}")" > "${local_path}"
  fi
}
wait_for_ssh() {
  local ip="$1"
  for _ in $(seq 1 120); do
    if ssh "${SSH_OPTS[@]}" "root@${ip}" true >/dev/null 2>&1; then
      return 0
    fi
    sleep 2
  done
  return 1
}
wait_for_node_ready() {
  local name="$1"
  for _ in $(seq 1 240); do
    if server_ssh "kubectl --kubeconfig /etc/kubernetes/admin.conf get node ${name} --no-headers 2>/dev/null | awk '\$2==\"Ready\"{ok=1} END{exit(ok?0:1)}'"; then
      return 0
    fi
    sleep 2
  done
  return 1
}

Скачивание ядра и rootfs

Используется образ ядра LinuxKit и последняя Ubuntu squashfs из Firecracker CI.

export LINUXKIT_KEY="$(printf '%s\n' "${LINUXKIT_KERNEL_IMAGE}" | sha256sum | awk '{print substr($1,1,16)}')"
export LINUXKIT_CACHE_DIR="${CACHE_ROOT}/downloads/linuxkit-${LINUXKIT_KEY}"
export KERNEL_PATH="${LINUXKIT_CACHE_DIR}/vmlinux"
export KERNEL_BZIMAGE_PATH="${LINUXKIT_CACHE_DIR}/kernel"
export KERNEL_MODULES_TAR_PATH="${LINUXKIT_CACHE_DIR}/kernel.tar"
export KERNEL_DEV_TAR_PATH="${LINUXKIT_CACHE_DIR}/kernel-dev.tar"
mkdir -p "${LINUXKIT_CACHE_DIR}"
if [[ ! -f "${KERNEL_PATH}" || ! -f "${KERNEL_MODULES_TAR_PATH}" ]]; then
  rm -rf "${LINUXKIT_CACHE_DIR}.tmp"
  mkdir -p "${LINUXKIT_CACHE_DIR}.tmp"
  docker pull "${LINUXKIT_KERNEL_IMAGE}" >/dev/null
  cid="$(docker create "${LINUXKIT_KERNEL_IMAGE}" /bin/sh)"
  docker cp "${cid}:/kernel" "${LINUXKIT_CACHE_DIR}.tmp/kernel"
  docker cp "${cid}:/kernel.tar" "${LINUXKIT_CACHE_DIR}.tmp/kernel.tar"
  docker cp "${cid}:/kernel-dev.tar" "${LINUXKIT_CACHE_DIR}.tmp/kernel-dev.tar"
  docker rm -f "${cid}" >/dev/null
  export LINUXKIT_HEADERS_DIR="$(tar -tf "${LINUXKIT_CACHE_DIR}.tmp/kernel-dev.tar" \
    | sed -n 's#^\(usr/src/linux-headers-[^/]*\)/scripts/extract-vmlinux$#\1#p' \
    | head -n 1)"
  tar -xOf "${LINUXKIT_CACHE_DIR}.tmp/kernel-dev.tar" \
    "${LINUXKIT_HEADERS_DIR}/scripts/extract-vmlinux" \
    > "${LINUXKIT_CACHE_DIR}.tmp/extract-vmlinux"
  chmod +x "${LINUXKIT_CACHE_DIR}.tmp/extract-vmlinux"
  "${LINUXKIT_CACHE_DIR}.tmp/extract-vmlinux" \
    "${LINUXKIT_CACHE_DIR}.tmp/kernel" \
    > "${LINUXKIT_CACHE_DIR}.tmp/vmlinux"
  mv "${LINUXKIT_CACHE_DIR}.tmp/kernel" "${KERNEL_BZIMAGE_PATH}"
  mv "${LINUXKIT_CACHE_DIR}.tmp/vmlinux" "${KERNEL_PATH}"
  mv "${LINUXKIT_CACHE_DIR}.tmp/kernel.tar" "${KERNEL_MODULES_TAR_PATH}"
  mv "${LINUXKIT_CACHE_DIR}.tmp/kernel-dev.tar" "${KERNEL_DEV_TAR_PATH}"
  rm -rf "${LINUXKIT_CACHE_DIR}.tmp"
fi

export FIRECRACKER_CI_VERSION="$("${FIRECRACKER_BIN}" --version | awk '{print $2}' | sed 's/\.[0-9]*$//')"
export UBUNTU_KEY="$(curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min?prefix=firecracker-ci/${FIRECRACKER_CI_VERSION}/${FIRECRACKER_ARCH}/ubuntu-&list-type=2" \
  | grep -oP "(?<=<Key>)(firecracker-ci/${FIRECRACKER_CI_VERSION}/${FIRECRACKER_ARCH}/ubuntu-[0-9]+\.[0-9]+\.squashfs)(?=</Key>)" \
  | sort -V | tail -1)"
export ROOTFS_SQUASHFS_PATH="${CACHE_ROOT}/downloads/$(basename "${UBUNTU_KEY}")"
mkdir -p "${CACHE_ROOT}/downloads"
if [[ ! -f "${ROOTFS_SQUASHFS_PATH}" ]]; then
  curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${UBUNTU\\_KEY}" -o "${ROOTFS_SQUASHFS_PATH}"
fi

Базовая ext4 rootfs

Преобразует squashfs в образ ext4 и внедряет гостевой SSH-ключ.

export BASE_KEY="$({ sha256sum "${ROOTFS_SQUASHFS_PATH}"; sha256sum "${GUEST_SSH_PUB}"; } | sha256sum | awk '{print substr($1,1,16)}')"
export BASE_ROOTFS_PATH="${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.ext4"
mkdir -p "${CACHE_ROOT}/base"
if [[ ! -s "${BASE_ROOTFS_PATH}" ]]; then
  rm -rf "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs"
  rm -f "${BASE_ROOTFS_PATH}.tmp"
  mkdir -p "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs"
  unsquashfs -d "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs" \
    "${ROOTFS_SQUASHFS_PATH}" >/dev/null
  mkdir -p "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs/root/.ssh"
  chmod 700 "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs/root/.ssh"
  cp "${GUEST_SSH_PUB}" \
    "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs/root/.ssh/authorized_keys"
  chmod 600 "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs/root/.ssh/authorized_keys"
  truncate -s 4G "${BASE_ROOTFS_PATH}.tmp"
  mkfs.ext4 -d "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs/rootfs" \
    -F "${BASE_ROOTFS_PATH}.tmp" >/dev/null
  mv "${BASE_ROOTFS_PATH}.tmp" "${BASE_ROOTFS_PATH}"
  rm -rf "${CACHE_ROOT}/base/ubuntu-${BASE_KEY}.rootfs"
fi

Гостевой образ с kubeadm

Устанавливает: модули ядра, containerd, kubelet, kubeadm, kubectl, плагины CNI, системные параметры Kubernetes и базовую настройку служб.

export PREPARED_KEY="$({ sha256sum "${BASE_ROOTFS_PATH}" "${KERNEL_PATH}" "${GUEST_SSH_PUB}" "${KERNEL_MODULES_TAR_PATH}"; printf 'kubernetes_minor=%s\n' "${KUBERNETES_MINOR}"; printf 'kubernetes_version=%s\n' "${KUBERNETES_VERSION}"; printf 'cni_plugins=%s\n' "${CNI_PLUGINS_VERSION}"; printf 'rootfs_size_gib=%s\n' "${ROOTFS_SIZE_GIB}"; printf 'generation=manual-kubeadm-firecracker-ha-v1\n'; } | sha256sum | awk '{print substr($1,1,16)}')"
export PREPARED_ROOTFS_PATH="${CACHE_ROOT}/prepared-${PREPARED_KEY}.ext4"
if [[ ! -s "${PREPARED_ROOTFS_PATH}" ]]; then
  cp --reflink=auto "${BASE_ROOTFS_PATH}" "${PREPARED_ROOTFS_PATH}.tmp" \
    2>/dev/null || cp "${BASE_ROOTFS_PATH}" "${PREPARED_ROOTFS_PATH}.tmp"
  e2fsck -fy "${PREPARED_ROOTFS_PATH}.tmp" >/dev/null 2>&1 || true
  truncate -s "${ROOTFS_SIZE_GIB}G" "${PREPARED_ROOTFS_PATH}.tmp"
  resize2fs "${PREPARED_ROOTFS_PATH}.tmp" >/dev/null
  export MNT="${CACHE_ROOT}/mnt-${PREPARED_KEY}"
  mkdir -p "${MNT}"
  mount -o loop "${PREPARED_ROOTFS_PATH}.tmp" "${MNT}"
  mount -t proc proc "${MNT}/proc"
  mount -t sysfs sysfs "${MNT}/sys"
  mount --bind /dev "${MNT}/dev"
  mount -t devpts devpts "${MNT}/dev/pts"
  mount --bind /run "${MNT}/run"
  rm -f "${MNT}/etc/resolv.conf"
  printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > "${MNT}/etc/resolv.conf"
  rm -rf "${MNT}/lib/modules"
  mkdir -p "${MNT}/lib"
  tar -xf "${KERNEL_MODULES_TAR_PATH}" -C "${MNT}" ./lib/modules
  chmod 1777 "${MNT}/tmp"
  mkdir -p "${MNT}/dev/pts" "${MNT}/var/cache/apt/archives/partial" \
    "${MNT}/var/lib/apt/lists/partial" "${MNT}/var/log/apt"
  touch "${MNT}/var/log/dpkg.log"
  mkdir -p "${MNT}/root/.ssh" && chmod 700 "${MNT}/root/.ssh"
  cp "${GUEST_SSH_PUB}" "${MNT}/root/.ssh/authorized_keys"
  chmod 600 "${MNT}/root/.ssh/authorized_keys"
  chroot "${MNT}" apt-get update
  DEBIAN_FRONTEND=noninteractive chroot "${MNT}" apt-get install -y \
    --no-install-recommends apt-transport-https ca-certificates conntrack \
    curl ebtables ethtool gpg iptables ipset jq openssh-server socat tar xz-utils
  mkdir -p "${MNT}/etc/apt/keyrings" "${MNT}/etc/apt/sources.list.d"
  docker_arch="$(chroot "${MNT}" dpkg --print-architecture)"
  docker_codename="$(awk -F= '/^VERSION_CODENAME=/{print $2}' "${MNT}/etc/os-release")"
  chroot "${MNT}" curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" \
    | chroot "${MNT}" gpg --dearmor -o /etc/apt/keyrings/docker.gpg
  printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu %s stable\n' \
    "${docker_arch}" "${docker_codename}" > "${MNT}/etc/apt/sources.list.d/docker.list"
  chroot "${MNT}" curl -fsSL "https://pkgs.k8s.io/core:/stable:/${KUBERNETES_MINOR}/deb/Release.key" \
    | chroot "${MNT}" gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
  printf 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/%s/deb/ /\n' \
    "${KUBERNETES_MINOR}" > "${MNT}/etc/apt/sources.list.d/kubernetes.list"
  chroot "${MNT}" apt-get update
  DEBIAN_FRONTEND=noninteractive chroot "${MNT}" apt-get install -y \
    containerd.io "kubelet=${KUBERNETES_VERSION#v}-*" \
    "kubeadm=${KUBERNETES_VERSION#v}-*" "kubectl=${KUBERNETES_VERSION#v}-*"
  chroot "${MNT}" apt-mark hold kubelet kubeadm kubectl >/dev/null 2>&1 || true
  mkdir -p "${MNT}/opt/cni/bin"
  tar -C "${MNT}/opt/cni/bin" -xzf "${CNI_ARCHIVE}"
  mkdir -p "${MNT}/etc/containerd"
  chroot "${MNT}" containerd config default > "${MNT}/etc/containerd/config.toml"
  sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' "${MNT}/etc/containerd/config.toml"
  cat > "${MNT}/etc/modules-load.d/kubernetes.conf" <<'EOF'
overlay
br_netfilter
EOF
  cat > "${MNT}/etc/sysctl.d/99-kubernetes.conf" <<'EOF'
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
EOF
  chroot "${MNT}" update-alternatives --set iptables /usr/sbin/iptables-legacy
  chroot "${MNT}" update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
  umount "${MNT}/dev/pts" || true
  umount "${MNT}/proc" || true
  umount "${MNT}/sys" || true
  umount "${MNT}/dev" || true
  umount "${MNT}/run" || true
  umount "${MNT}" || true
  mv "${PREPARED_ROOTFS_PATH}.tmp" "${PREPARED_ROOTFS_PATH}"
fi

Мост, NAT и HAProxy

Явные правила FORWARD важны на хостах, где политика FORWARD по умолчанию — DROP.

docker rm -f "${API_LB_CONTAINER_NAME}" >/dev/null 2>&1 || true
ip link add name "${BRIDGE_NAME}" type bridge 2>/dev/null || true
ip addr add "${GATEWAY}/24" dev "${BRIDGE_NAME}" 2>/dev/null || true
ip addr add "${API_LB_IP}/24" dev "${BRIDGE_NAME}" 2>/dev/null || true
ip link set "${BRIDGE_NAME}" up
sysctl -w net.ipv4.ip_forward=1 >/dev/null
iptables -C FORWARD -i "${BRIDGE_NAME}" -j ACCEPT 2>/dev/null \
  || iptables -A FORWARD -i "${BRIDGE_NAME}" -j ACCEPT
iptables -C FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null \
  || iptables -A FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
iptables -t nat -C POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE 2>/dev/null \
  || iptables -t nat -A POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE

cat > "${RUN_ROOT}/haproxy.cfg" <<EOF
global
  maxconn 2048
defaults
  mode tcp
  timeout connect 5s
  timeout client 60s
  timeout server 60s
frontend kube_api
  bind ${API_LB_IP}:${API_LB_PORT}
  default_backend kube_apis
backend kube_apis
  balance roundrobin
  option tcp-check
  default-server inter 2s fall 3 rise 2
EOF
for i in $(seq 0 "$((CONTROL_PLANE_COUNT - 1))"); do
  printf '  server cp%s %s:6443 check\n' "${i}" "$(node_ip "${i}")" \
    >> "${RUN_ROOT}/haproxy.cfg"
done
docker run -d --name "${API_LB_CONTAINER_NAME}" --network host \
  -v "${RUN_ROOT}/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" \
  "${HAPROXY_IMAGE}" >/dev/null

Создание микро-ВМ

Каждая ВМ получает клонированный диск ext4, имя хоста, /etc/hosts и конфигурацию systemd-networkd.

for i in $(seq 0 "$((NODE_COUNT - 1))"); do
  vm_dir="${RUN_ROOT}/nodes/node${i}"
  mkdir -p "${vm_dir}"
  cp --reflink=auto "${PREPARED_ROOTFS_PATH}" "${vm_dir}/rootfs.ext4" \
    2>/dev/null || cp "${PREPARED_ROOTFS_PATH}" "${vm_dir}/rootfs.ext4"
  e2fsck -fy "${vm_dir}/rootfs.ext4" >/dev/null 2>&1 || true
  mkdir -p "${vm_dir}/mnt"
  mount -o loop "${vm_dir}/rootfs.ext4" "${vm_dir}/mnt"
  printf '%s\n' "$(node_name "${i}")" > "${vm_dir}/mnt/etc/hostname"
  {
    printf '127.0.0.1 localhost\n'
    printf '127.0.1.1 %s\n' "$(node_name "${i}")"
    printf '%s api-lb\n' "${API_LB_IP}"
    for j in $(seq 0 "$((NODE_COUNT - 1))"); do
      printf '%s %s\n' "$(node_ip "${j}")" "$(node_name "${j}")"
    done
  } > "${vm_dir}/mnt/etc/hosts"
  mkdir -p "${vm_dir}/mnt/etc/systemd/network"
  cat > "${vm_dir}/mnt/etc/systemd/network/20-eth0.network" <<EOF
[Match]
Name=eth0
[Network]
Address=$(node_ip "${i}")/24
Gateway=${GATEWAY}
DNS=1.1.1.1
DNS=8.8.8.8
EOF
  rm -f "${vm_dir}/mnt/etc/machine-id"
  touch "${vm_dir}/mnt/etc/machine-id"
  rm -rf "${vm_dir}/mnt/etc/kubernetes" "${vm_dir}/mnt/var/lib/etcd"
  mkdir -p "${vm_dir}/mnt/var/lib/containerd" "${vm_dir}/mnt/etc/cni/net.d"
  umount "${vm_dir}/mnt"
  tap="$(node_tap "${i}")"
  mac="$(node_mac "${i}")"
  mem="$(node_mem "${i}")"
  ip tuntap add dev "${tap}" mode tap 2>/dev/null || true
  ip link set "${tap}" master "${BRIDGE_NAME}"
  ip link set "${tap}" up
  cat > "${vm_dir}/vm.json" <<EOF
{"boot-source":{"kernel_image_path":"${KERNEL_PATH}","boot_args":"${KERNEL_BOOT_ARGS}"},"drives":[{"drive_id":"rootfs","path_on_host":"${vm_dir}/rootfs.ext4","is_root_device":true,"is_read_only":false}],"machine-config":{"vcpu_count":${VCPU_COUNT},"mem_size_mib":${mem}},"network-interfaces":[{"iface_id":"eth0","host_dev_name":"${tap}","guest_mac":"${mac}"}],"logger":{"log_path":"${vm_dir}/firecracker.log","level":"Info","show_level":true,"show_log_origin":true}}
EOF
  "${FIRECRACKER_BIN}" --api-sock "${vm_dir}/fc.sock" \
    --config-file "${vm_dir}/vm.json" > "${vm_dir}/console.log" 2>&1 &
  echo $! > "${vm_dir}/pid"
done

Подготовка узлов

Ожидание SSH и подготовка узлов.

for i in $(seq 0 "$((NODE_COUNT - 1))"); do
  wait_for_ssh "$(node_ip "${i}")"
done
for i in $(seq 0 "$((NODE_COUNT - 1))"); do
  node_ssh "${i}" "cat >/root/prepare-node.sh" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
swapoff -a || true
sed -i '/ swap / s/^/#/' /etc/fstab 2>/dev/null || true
modprobe overlay || true
modprobe br_netfilter || true
sysctl --system >/dev/null
systemctl daemon-reload || true
systemctl enable --now ssh containerd kubelet >/dev/null 2>&1 || true
systemctl restart containerd
kubeadm reset -f >/dev/null 2>&1 || true
rm -rf /etc/kubernetes /var/lib/etcd /var/lib/cni /var/lib/kubelet/* /etc/cni/net.d/*
mkdir -p /etc/cni/net.d
kubeadm config images pull --kubernetes-version="$(kubeadm version -o short)" >/dev/null 2>&1 || true
EOF
  node_ssh "${i}" "chmod +x /root/prepare-node.sh && /root/prepare-node.sh"
done

Инициализация control-plane

server_ssh "cat >/root/init-primary.sh" <<EOF
#!/usr/bin/env bash
set -euo pipefail
version="${KUBERNETES_VERSION}"
kubeadm init \
  --kubernetes-version "\${version}" \
  --control-plane-endpoint "${API_LB_IP}:${API_LB_PORT}" \
  --apiserver-advertise-address "${PRIMARY_CONTROL_PLANE_IP}" \
  --pod-network-cidr "${POD_CIDR}" \
  --service-cidr "${SERVICE_CIDR}" \
  --upload-certs \
  --ignore-preflight-errors=all
mkdir -p /root/.kube
cp /etc/kubernetes/admin.conf /root/.kube/config
certificate_key="\$(kubeadm init phase upload-certs --upload-certs 2>/dev/null | tail -n 1)"
join_cmd="\$(kubeadm token create --print-join-command)"
printf '%s\n' "\${certificate_key}" >/root/certificate-key.txt
printf '%s\n' "\${join_cmd}" >/root/join-command.txt
EOF
server_ssh "chmod +x /root/init-primary.sh && /root/init-primary.sh"

Установка Flannel

server_ssh "kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f $(printf '%q' "${FLANNEL_MANIFEST_URL}")"
for _ in $(seq 1 180); do
  flannel_ns="$(server_ssh "kubectl --kubeconfig /etc/kubernetes/admin.conf get daemonset -A --no-headers 2>/dev/null | awk '\$2==\"kube-flannel-ds\"{print \$1; exit}'" || true)"
  if [[ -n "${flannel_ns}" ]]; then
    if server_ssh "kubectl --kubeconfig /etc/kubernetes/admin.conf -n ${flannel_ns} rollout status daemonset/kube-flannel-ds --timeout=5s" >/dev/null 2>&1; then
      break
    fi
  fi
  sleep 2
done

Подключение узлов

join_cmd="$(server_ssh "cat /root/join-command.txt")"
certificate_key="$(server_ssh "cat /root/certificate-key.txt")"

# Control-plane узлы
for i in 1 2; do
  ip="$(node_ip "${i}")"
  node_ssh "${i}" "cat >/root/join-control-plane.sh" <<EOF
#!/usr/bin/env bash
set -euo pipefail
${join_cmd} --control-plane --certificate-key ${certificate_key} --apiserver-advertise-address ${ip} --ignore-preflight-errors=all
EOF
  node_ssh "${i}" "chmod +x /root/join-control-plane.sh && /root/join-control-plane.sh"
  wait_for_node_ready "$(node_name "${i}")"
done

# Worker-узлы
for i in $(seq "${CONTROL_PLANE_COUNT}" "$((NODE_COUNT - 1))"); do
  node_ssh "${i}" "cat >/root/join-worker.sh" <<EOF
#!/usr/bin/env bash
set -euo pipefail
${join_cmd} --ignore-preflight-errors=all
EOF
  node_ssh "${i}" "chmod +x /root/join-worker.sh && /root/join-worker.sh"
  wait_for_node_ready "$(node_name "${i}")"
done

Готовность кластера

Ожидаемый результат:

k8s-00   Ready   control-plane
k8s-01   Ready   control-plane
k8s-02   Ready   control-plane
k8s-03   Ready   <none>
k8s-04   Ready   <none>

Проверка etcd

Должно отобразиться три участника etcd.

etcd_pod="etcd-$(node_name 0)"
server_ssh "kubectl --kubeconfig /etc/kubernetes/admin.conf \
  -n kube-system exec ${etcd_pod} -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
  --key=/etc/kubernetes/pki/etcd/healthcheck-client.key \
  member list -w table"

Smoke-тест

server_ssh "cat >/root/run-smoke.sh" <<'EOSMOKE'
#!/usr/bin/env bash
set -euo pipefail
kubectl --kubeconfig /etc/kubernetes/admin.conf create namespace smoke \
  --dry-run=client -o yaml | kubectl apply -f -

kubectl apply -f - <<'YAML'
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-smoke
  namespace: smoke
spec:
  selector:
    matchLabels:
      app: node-smoke
  template:
    metadata:
      labels:
        app: node-smoke
    spec:
      tolerations:
        - operator: Exists
      containers:
        - name: node-smoke
          image: busybox:1.36
          command: ["sh", "-lc", "sleep 3600"]
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: echo
  namespace: smoke
spec:
  replicas: 2
  selector:
    matchLabels:
      app: echo
  template:
    metadata:
      labels:
        app: echo
    spec:
      tolerations:
        - operator: Exists
      containers:
        - name: echo
          image: nginx:1.27-alpine
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: echo
  namespace: smoke
spec:
  selector:
    app: echo
  ports:
    - port: 80
      targetPort: 80
YAML

kubectl -n smoke rollout status daemonset/node-smoke --timeout=240s
kubectl -n smoke rollout status deployment/echo --timeout=240s
kubectl -n kube-system rollout status deployment/coredns --timeout=240s

kubectl apply -f - <<'YAML'
apiVersion: batch/v1
kind: Job
metadata:
  name: dns-http
  namespace: smoke
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      tolerations:
        - operator: Exists
      containers:
        - name: dns-http
          image: busybox:1.36
          command:
            - sh
            - -lc
            - |
              for _ in $(seq 1 30); do
                nslookup echo.smoke.svc.cluster.local && \
                wget -qO- http://echo.smoke.svc.cluster.local >/tmp/echo.html && \
                test -s /tmp/echo.html && exit 0
                sleep 2
              done
              exit 1
YAML
kubectl -n smoke wait --for=condition=complete job/dns-http --timeout=240s
kubectl get nodes -o wide > /var/log/kubeadm-ha-lab.nodes 2>&1
kubectl get pods -A -o wide > /var/log/kubeadm-ha-lab.pods 2>&1
kubectl -n smoke get pods -o wide > /var/log/kubeadm-ha-lab.smoke-pods 2>&1
kubectl -n smoke logs job/dns-http > /var/log/kubeadm-ha-lab.smoke-job.log 2>&1
kubectl config view --raw > /var/log/kubeadm-ha-lab.kubeconfig 2>&1
EOSMOKE
server_ssh "chmod +x /root/run-smoke.sh && /root/run-smoke.sh"

Сохранение артефактов

export KUBECONFIG="${RUN_ROOT}/artifacts/kubeconfig.yaml"
kubectl get nodes -o wide
kubectl get pods -A -o wide

Демонтаж

Остановка Firecracker, удаление tap, правил iptables, моста и RUN_ROOT. CACHE_ROOT сохраняется для переиспользования.

docker rm -f "${API_LB_CONTAINER_NAME}" >/dev/null 2>&1 || true
for pid_file in "${RUN_ROOT}"/nodes/*/pid; do
  [[ -f "${pid_file}" ]] || continue
  kill "$(cat "${pid_file}")" 2>/dev/null || true
done
sleep 1
for pid_file in "${RUN_ROOT}"/nodes/*/pid; do
  [[ -f "${pid_file}" ]] || continue
  kill -9 "$(cat "${pid_file}")" 2>/dev/null || true
done
for i in $(seq 0 "$((NODE_COUNT - 1))"); do
  ip link del "$(node_tap "${i}")" 2>/dev/null || true
done
iptables -D FORWARD -i "${BRIDGE_NAME}" -j ACCEPT 2>/dev/null || true
iptables -D FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
iptables -t nat -D POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE 2>/dev/null || true
ip link set "${BRIDGE_NAME}" down 2>/dev/null || true
ip link del "${BRIDGE_NAME}" type bridge 2>/dev/null || true
rm -rf "${RUN_ROOT}"

Создай свой Talos или CoreOS Linux I

Антон Крылов | 6 мин чтения | 29 мая 2026 г.

В отличие от предыдущего руководства с полным HA-кластером (3 control-plane, kubeadm, etcd, HAProxy, ext4 rootfs на 12 ГБ), здесь подход намеренно минималистичен: k3s вместо kubeadm, initrd вместо тяжёлой корневой файловой системы, один control-plane вместо трёх. Никакого балансировщика, никакого внешнего etcd — только ядро, initrd и контейнеры. Это чертёж не прод-кластера, а идеи: Kubernetes-ОС может помещаться в оперативной памяти.

Talos показывает, что ОС, ориентированная на Kubernetes, может быть крошечной, неизменяемой и целенаправленной — так же, как CoreOS когда-то делала для контейнерных хостов. Но более интересная идея в том, что этот подход не является магией или прерогативой одного вендора. С помощью таких инструментов, как LinuxKit, минимального загрузочного образа и компактной среды выполнения Kubernetes, вы можете собрать собственную урезанную ОС для узлов: настраиваемую, повторяемую и содержащую только те компоненты, которые действительно нужны вашему кластеру.

Предварительные требования (Talos)

Выполняйте всё от root на хосте Linux, где будет запущен Firecracker. На хосте необходимы:

Docker
Firecracker
ip, iptables, ssh, ssh-keygen, openssl
исходящий доступ в интернет для загрузки контейнерных образов

Выберите подсеть, не пересекающуюся с другими мостами на хосте.

1. Экспорт переменных

export RUN_ROOT=/var/lib/create-your-own-talos
export NODE_COUNT=3
export SUBNET_PREFIX=198.19.0
export SERVER_IP="${SUBNET_PREFIX}.10"
export GATEWAY="${SUBNET_PREFIX}.1"
export CIDR="${SUBNET_PREFIX}.0/24"
export BRIDGE_NAME=lkt198
export TAP_PREFIX=lkt198
export FIRECRACKER_BIN=/usr/local/bin/firecracker
export LINUXKIT_VERSION=v1.8.2
export LINUXKIT_BIN=/root/linuxkit/bin/linuxkit
export LINUXKIT_KERNEL_IMAGE=linuxkit/kernel:6.12.59
export K3S_IMAGE=rancher/k3s:v1.36.1-k3s1
export K3S_TOKEN="$(openssl rand -hex 24)"
export GUEST_SSH_KEY="${RUN_ROOT}/lab_ssh_key"
export GUEST_SSH_PUB="${GUEST_SSH_KEY}.pub"

2. LinuxKit и SSH

command -v docker ip iptables ssh ssh-keygen openssl >/dev/null
test -x "${FIRECRACKER_BIN}"
mkdir -p "$(dirname "${LINUXKIT_BIN}")" "${RUN_ROOT}"
if [[ ! -x "${LINUXKIT_BIN}" ]]; then
  curl -fsSL \
    "https://github.com/linuxkit/linuxkit/releases/download/${LINUXKIT_VERSION}/linuxkit-linux-amd64" \
    -o "${LINUXKIT_BIN}"
  chmod +x "${LINUXKIT_BIN}"
fi
if [[ ! -f "${GUEST_SSH_KEY}" || ! -f "${GUEST_SSH_PUB}" ]]; then
  ssh-keygen -q -t ed25519 -N "" -f "${GUEST_SSH_KEY}"
fi

3. Вспомогательные функции

node_ip() {
  printf '%s.%d\n' "${SUBNET_PREFIX}" "$((10 + $1))"
}
node_name() {
  printf 'linuxkit-%02d\n' "$1"
}
node_tap() {
  printf '%s%d\n' "${TAP_PREFIX}" "$1"
}
node_mac() {
  printf '06:19:80:00:00:%02x\n' "$((16 + $1))"
}
node_mem() {
  if [[ "$1" == "0" ]]; then
    echo 2048
  else
    echo 1536
  fi
}
SSH_OPTS=(-n -i "${GUEST_SSH_KEY}" -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR)
server_ssh() {
  ssh "${SSH_OPTS[@]}" "root@${SERVER_IP}" "$@"
}
k3s_exec() {
  local cmd="$1"
  local exec_id="x$(date +%s%N)"
  server_ssh "ctr -n services.linuxkit tasks exec --exec-id ${exec_id} k3s /bin/sh -lc $(printf '%q' "${cmd}")"
}

4. LinuxKit YAML

Control plane: k3s server с --cluster-init. Рабочие узлы: k3s agent.

kernel:
  image: ${LINUXKIT_KERNEL_IMAGE}
  cmdline: "console=tty0 console=ttyS0 console=ttyAMA0"
init:
  - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a
  - linuxkit/runc:9442aa234715e751a16144f1d4ae3fd1a00fd492
  - linuxkit/containerd:ba19f64efd3331a8fd0a33e00eabd14f6ee1780e
  - linuxkit/ca-certificates:256f1950df59f2f209e9f0b81374177409eb11de
onboot:
  - name: sysctl
    image: linuxkit/sysctl:43ac1d39da329c3567fcb9689e5ca99de6d169b6
  - name: ip
    image: linuxkit/ip:3c0676ee83a0dc739685be1253b8326f08581ef7
    binds:
      - /etc/ip:/etc/ip
    command: ["ip", "-b", "/etc/ip/eth0.conf"]
services:
  - name: getty
    image: linuxkit/getty:a86d74c8f89be8956330c3b115b0b1f2e09ef6e0
    env:
      - INSECURE=true
  - name: rngd
    image: linuxkit/rngd:984eb580ecb63986f07f626b61692a97aacd7198
  - name: sshd
    image: linuxkit/sshd:08e5d4a46603eff485d5d1b14001cc932a530858
    binds.add:
      - /root/.ssh:/root/.ssh
  - name: k3s
    image: ${K3S_IMAGE}
    capabilities:
      - all
    net: host
    rootfsPropagation: shared
    devices:
      - path: /dev/kmsg
        type: c
        major: 1
        minor: 11
        mode: "0666"
    mounts:
      - type: cgroup
        options: ["rw","nosuid","noexec","nodev","relatime"]
    binds:
      - /etc/resolv.conf:/etc/resolv.conf
      - /lib/modules:/lib/modules
      - /var:/var:rshared,rbind
    env:
      - PATH=/bin:/bin/aux:/usr/bin:/sbin:/usr/sbin
      - K3S_TOKEN=${K3S_TOKEN}
    command:
      - /bin/k3s
      - server
      - --cluster-init
      - --node-name=linuxkit-00
      - --node-ip=${SERVER_IP}
      - --advertise-address=${SERVER_IP}
      - --bind-address=0.0.0.0
      - --tls-san=${SERVER_IP}
      - --write-kubeconfig-mode=0644
      - --disable=traefik
      - --disable=servicelb
      - --disable=metrics-server
      - --disable=local-storage
      - --disable-cloud-controller
      - --disable-network-policy
      - --flannel-iface=eth0
      - --flannel-backend=host-gw
files:
  - path: etc/ip/eth0.conf
    contents: |
      address add ${SERVER_IP}/24 dev eth0
      link set eth0 up
      route add default via ${GATEWAY} dev eth0
  - path: etc/resolv.conf
    contents: |
      nameserver 1.1.1.1
      nameserver 8.8.8.8
  - path: root/.ssh/authorized_keys
    source: ${GUEST_SSH_PUB}
    mode: "0600"

kernel:
  image: ${LINUXKIT_KERNEL_IMAGE}
  cmdline: "console=tty0 console=ttyS0 console=ttyAMA0"
init:
  - linuxkit/init:b5506cc74a6812dc40982cacfd2f4328f8a4b12a
  - linuxkit/runc:9442aa234715e751a16144f1d4ae3fd1a00fd492
  - linuxkit/containerd:ba19f64efd3331a8fd0a33e00eabd14f6ee1780e
  - linuxkit/ca-certificates:256f1950df59f2f209e9f0b81374177409eb11de
onboot:
  - name: sysctl
    image: linuxkit/sysctl:43ac1d39da329c3567fcb9689e5ca99de6d169b6
  - name: ip
    image: linuxkit/ip:3c0676ee83a0dc739685be1253b8326f08581ef7
    binds:
      - /etc/ip:/etc/ip
    command: ["ip", "-b", "/etc/ip/eth0.conf"]
services:
  - name: getty
    image: linuxkit/getty:a86d74c8f89be8956330c3b115b0b1f2e09ef6e0
    env:
      - INSECURE=true
  - name: rngd
    image: linuxkit/rngd:984eb580ecb63986f07f626b61692a97aacd7198
  - name: sshd
    image: linuxkit/sshd:08e5d4a46603eff485d5d1b14001cc932a530858
    binds.add:
      - /root/.ssh:/root/.ssh
  - name: k3s
    image: ${K3S_IMAGE}
    capabilities:
      - all
    net: host
    rootfsPropagation: shared
    devices:
      - path: /dev/kmsg
        type: c
        major: 1
        minor: 11
        mode: "0666"
    mounts:
      - type: cgroup
        options: ["rw","nosuid","noexec","nodev","relatime"]
    binds:
      - /etc/resolv.conf:/etc/resolv.conf
      - /lib/modules:/lib/modules
      - /var:/var:rshared,rbind
    env:
      - PATH=/bin:/bin/aux:/usr/bin:/sbin:/usr/sbin
      - K3S_TOKEN=${K3S_TOKEN}
    command:
      - /bin/k3s
      - agent
      - --server=https://${SERVER\\_IP}:6443
      - --node-name=__NODE_NAME__
      - --node-ip=__NODE_IP__
      - --flannel-iface=eth0
files:
  - path: etc/ip/eth0.conf
    contents: |
      address add __NODE_IP__/24 dev eth0
      link set eth0 up
      route add default via ${GATEWAY} dev eth0
  - path: etc/resolv.conf
    contents: |
      nameserver 1.1.1.1
      nameserver 8.8.8.8
  - path: root/.ssh/authorized_keys
    source: ${GUEST_SSH_PUB}
    mode: "0600"

5. Сборка образов

mkdir -p "${RUN_ROOT}/nodes/node0"
cp "${RUN_ROOT}/server.yml" "${RUN_ROOT}/nodes/node0/node.yml"
"${LINUXKIT_BIN}" build \
  --decompress-kernel \
  --dir "${RUN_ROOT}/nodes/node0" \
  --name node \
  -f kernel+initrd \
  "${RUN_ROOT}/nodes/node0/node.yml"
for i in 1 2; do
  ip="$(node_ip "${i}")"
  name="$(node_name "${i}")"
  dir="${RUN_ROOT}/nodes/node${i}"
  mkdir -p "${dir}"
  sed \
    -e "s/__NODE_NAME__/${name}/g" \
    -e "s/__NODE_IP__/${ip}/g" \
    "${RUN_ROOT}/agent.yml.tmpl" > "${dir}/node.yml"
  "${LINUXKIT_BIN}" build \
    --decompress-kernel \
    --dir "${dir}" \
    --name node \
    -f kernel+initrd \
    "${dir}/node.yml"
done

6. Мост и NAT

ip link add name "${BRIDGE_NAME}" type bridge 2>/dev/null || true
ip addr add "${GATEWAY}/24" dev "${BRIDGE_NAME}" 2>/dev/null || true
ip link set "${BRIDGE_NAME}" up
sysctl -w net.ipv4.ip_forward=1
iptables -C FORWARD -i "${BRIDGE_NAME}" -j ACCEPT 2>/dev/null || iptables -A FORWARD -i "${BRIDGE_NAME}" -j ACCEPT
iptables -C FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || \
  iptables -A FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
iptables -t nat -C POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE 2>/dev/null || \
  iptables -t nat -A POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE

7. Загрузка микро-ВМ

for i in 0 1 2; do
  dir="${RUN_ROOT}/nodes/node${i}"
  tap="$(node_tap "${i}")"
  mac="$(node_mac "${i}")"
  mem="$(node_mem "${i}")"
  ip tuntap add dev "${tap}" mode tap 2>/dev/null || true
  ip link set "${tap}" master "${BRIDGE_NAME}"
  ip link set "${tap}" up
  cat > "${dir}/vm.json" <<EOF
{"boot-source":{"kernel_image_path":"${dir}/node-kernel","initrd_path":"${dir}/node-initrd.img","boot_args":"$(tr -d '\n' <"${dir}/node-cmdline") reboot=k panic=1 pci=off random.trust_cpu=on"},"drives":[],"machine-config":{"vcpu_count":2,"mem_size_mib":${mem}},"network-interfaces":[{"iface_id":"eth0","host_dev_name":"${tap}","guest_mac":"${mac}"}],"logger":{"log_path":"${dir}/firecracker.log","level":"Info","show_level":true,"show_log_origin":true}}
EOF
  "${FIRECRACKER_BIN}" \
    --api-sock "${dir}/fc.sock" \
    --config-file "${dir}/vm.json" \
    > "${dir}/console.log" 2>&1 &
  echo $! > "${dir}/pid"
done

8. Готовность кластера

for ip in "$(node_ip 0)" "$(node_ip 1)" "$(node_ip 2)"; do
  until ssh "${SSH_OPTS[@]}" "root@${ip}" true >/dev/null 2>&1; do
    sleep 2
  done
done
for _ in $(seq 1 180); do
  k3s_exec "kubectl get nodes -o wide > /var/log/create-your-own-talos.nodes 2>&1 || true" >/dev/null 2>&1 || true
  ready_count="$(server_ssh "awk '\$2==\"Ready\"{c++} END{print c+0}' /var/log/create-your-own-talos.nodes 2>/dev/null || echo 0")"
  if [[ "${ready_count}" -ge "${NODE_COUNT}" ]]; then
    break
  fi
  sleep 3
done
server_ssh "cat /var/log/create-your-own-talos.nodes"

Ожидаемый результат:

linuxkit-00   Ready   control-plane,etcd
linuxkit-01   Ready   <none>
linuxkit-02   Ready   <none>

9. Smoke-тест

k3s_exec "$(cat <<'EOF'
kubectl create namespace smoke --dry-run=client -o yaml | kubectl apply -f -
cat <<'YAML' | kubectl apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-smoke
  namespace: smoke
spec:
  selector:
    matchLabels:
      app: node-smoke
  template:
    metadata:
      labels:
        app: node-smoke
    spec:
      tolerations:
        - operator: Exists
      containers:
        - name: node-smoke
          image: busybox:1.36
          command: ["sh", "-lc", "sleep 3600"]
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: echo
  namespace: smoke
spec:
  replicas: 2
  selector:
    matchLabels:
      app: echo
  template:
    metadata:
      labels:
        app: echo
    spec:
      containers:
        - name: echo
          image: nginx:1.27-alpine
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: echo
  namespace: smoke
spec:
  selector:
    app: echo
  ports:
    - port: 80
      targetPort: 80
YAML
kubectl -n smoke rollout status daemonset/node-smoke --timeout=240s
kubectl -n smoke rollout status deployment/echo --timeout=240s
kubectl -n kube-system rollout status deployment/coredns --timeout=240s
cat <<'YAML' | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
  name: dns-http
  namespace: smoke
spec:
  backoffLimit: 0
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: dns-http
          image: busybox:1.36
          command:
            - sh
            - -lc
            - |
              for _ in $(seq 1 30); do
                nslookup echo.smoke.svc.cluster.local && \
                wget -qO- http://echo.smoke.svc.cluster.local >/tmp/echo.html && \
                test -s /tmp/echo.html && exit 0
                sleep 2
              done
              exit 1
YAML
kubectl -n smoke wait --for=condition=complete job/dns-http --timeout=240s
kubectl get pods -A -o wide > /var/log/create-your-own-talos.pods 2>&1
kubectl -n smoke get pods -o wide > /var/log/create-your-own-talos.smoke-pods 2>&1
kubectl -n smoke logs job/dns-http > /var/log/create-your-own-talos.smoke-job.log 2>&1
kubectl config view --raw > /var/log/create-your-own-talos.kubeconfig 2>&1
EOF
)"

10. Сохранение артефактов

export KUBECONFIG="${RUN_ROOT}/artifacts/kubeconfig.yaml"
kubectl get nodes -o wide
kubectl get pods -A -o wide

11. Демонтаж

for pid_file in "${RUN_ROOT}"/nodes/*/pid; do
  [[ -f "${pid_file}" ]] || continue
  kill "$(cat "${pid_file}")" 2>/dev/null || true
done
sleep 1
for pid_file in "${RUN_ROOT}"/nodes/*/pid; do
  [[ -f "${pid_file}" ]] || continue
  kill -9 "$(cat "${pid_file}")" 2>/dev/null || true
done
for i in 0 1 2; do
  ip link del "$(node_tap "${i}")" 2>/dev/null || true
done
iptables -D FORWARD -i "${BRIDGE_NAME}" -j ACCEPT 2>/dev/null || true
iptables -D FORWARD -o "${BRIDGE_NAME}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true
iptables -t nat -D POSTROUTING -s "${CIDR}" ! -o "${BRIDGE_NAME}" -j MASQUERADE 2>/dev/null || true
ip link set "${BRIDGE_NAME}" down 2>/dev/null || true
ip link del "${BRIDGE_NAME}" type bridge 2>/dev/null || true
rm -rf "${RUN_ROOT}"