diff --git a/ansible/README.md b/ansible/README.md index a3faa7d..8297f83 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -169,4 +169,12 @@ master 逐台升级,worker 保持并行(默认 20 forks)。已有 server 在安装 Ansible 的 Python 环境运行就绪检查回归:`python -m unittest discover -s tests -v`。 +### 2026-10-16 执行记录 +16 个节点已升级至 `v1.35.9+k3s1`,3 个 server 逐台执行,13 个 worker 并行执行。升级前的不压缩快照以 `pre-upgrade-v1.35.9+k3s1--` 保存在 snapshots 目录,恢复用 token 为 server 目录下的 `pre-upgrade-v1.35.9+k3s1.token`。 + +homea 的 Velero 镜像拉取曾停滞:确认两个现有镜像源能下载、磁盘充足且没有进程代理后,通过 `ansible homea -m ansible.builtin.systemd -a 'name=k3s-agent state=restarted' -e ansible_port=2103` 释放旧下载请求,DaemonSet 随后完成更新。Velero 的 Helm 等待因此超时,通过相同时间戳的 `reconcile.fluxcd.io/requestedAt` 和 `reconcile.fluxcd.io/resetAt` 注解重新协调成功。 + +Grafana 的大镜像层再次在该节点的 1ms 链路停滞,分段下载验证 Daocloud 可用后,通过 homea 的 `registry_mirror_overrides` 调整 Docker 镜像优先级;其他 registry 继承全局配置。镜像配置单独变化时,role 会重启对应 K3s 服务并等待 Ready;版本升级时由安装流程加载新配置。调整后 Grafana 镜像约 27 秒拉取完成;Prometheus 栈因旧下载触发的部署超时也通过上述注解重新协调成功。 + +CloudNativePG/Barman 升级重建了两套单实例 PostgreSQL Pod,恢复后均为 2/2 Ready;PostgreSQL 数据镜像保持 17.4。各执行一次 `SELECT pg_switch_wal();` 验证新插件,两个实例的归档计数均增长,归档失败计数未增加。未进行数据恢复演练。 diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index cfdab1c..030b04d 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -108,6 +108,11 @@ all: node_hostname: homea node_region: cn-sh use_mirror: true + # 此线路访问 1ms 的 Docker 镜像层会停滞,优先使用现有备用源。 + registry_mirror_overrides: + docker.io: + - docker.m.daocloud.io + - docker.1ms.run # 节点分组 k3s_cluster: diff --git a/ansible/roles/k3s/tasks/main.yml b/ansible/roles/k3s/tasks/main.yml index 86a67d5..7f31761 100644 --- a/ansible/roles/k3s/tasks/main.yml +++ b/ansible/roles/k3s/tasks/main.yml @@ -51,6 +51,7 @@ dest: /etc/rancher/k3s/registries.yaml mode: "0644" when: use_mirror | default(false) + register: k3s_registries_config # 判断是否需要安装/重启 # 使用 -e force_reinstall=true 可强制重新安装 @@ -136,6 +137,15 @@ path: /tmp/k3s-install.sh state: absent +- name: Restart K3s after updating registry mirrors + ansible.builtin.systemd: + name: "{{ 'k3s' if inventory_hostname in groups['masters'] else 'k3s-agent' }}" + state: restarted + when: + - k3s_registries_config.changed | default(false) + - k3s_binary.stat.exists + - k3s_version in k3s_current_version.stdout + # /readyz includes the local API server's etcd readiness check. - name: Wait for the local API server and etcd to be ready ansible.builtin.command: diff --git a/ansible/roles/k3s/templates/registries.yaml.j2 b/ansible/roles/k3s/templates/registries.yaml.j2 index e36f1c3..37eb080 100644 --- a/ansible/roles/k3s/templates/registries.yaml.j2 +++ b/ansible/roles/k3s/templates/registries.yaml.j2 @@ -1,7 +1,7 @@ # 镜像加速配置 --- mirrors: -{% for registry, endpoints in registry_mirrors.items() %} +{% for registry, endpoints in (registry_mirrors | combine(registry_mirror_overrides | default({}))).items() %} "{{ registry }}": endpoint: {% for endpoint in endpoints %} diff --git a/ansible/tests/test_readiness.py b/ansible/tests/test_readiness.py index 61edd9e..64e4939 100644 --- a/ansible/tests/test_readiness.py +++ b/ansible/tests/test_readiness.py @@ -3,6 +3,7 @@ import unittest from pathlib import Path import yaml +from ansible.plugins.filter.core import combine from jinja2 import Environment, StrictUndefined @@ -14,6 +15,9 @@ class ReadinessTest(unittest.TestCase): ) env = Environment(undefined=StrictUndefined) env.filters["from_json"] = json.loads + env.filters["combine"] = combine + cls.env = env + cls.tasks = tasks cls.conditions = { task["register"]: env.compile_expression(task["until"]) for task in tasks @@ -47,6 +51,37 @@ class ReadinessTest(unittest.TestCase): with self.subTest(rc=rc, stdout=stdout): self.assertEqual(condition(k3s_api_ready={"rc": rc, "stdout": stdout}), expected) + def test_mirror_changes_restart_existing_nodes_without_a_version_upgrade(self): + task = next(t for t in self.tasks if t["name"] == "Restart K3s after updating registry mirrors") + condition = self.env.compile_expression(" and ".join(f"({c})" for c in task["when"])) + for changed, exists, current, expected in [ + (True, True, "v1.35.9+k3s1", True), + (False, True, "v1.35.9+k3s1", False), + (True, True, "v1.35.6+k3s1", False), + (True, False, "", False), + ]: + with self.subTest(changed=changed, exists=exists, current=current): + self.assertEqual(condition( + k3s_registries_config={"changed": changed}, + k3s_binary={"stat": {"exists": exists}}, + k3s_current_version={"stdout": current} if exists else {}, + k3s_version="v1.35.9+k3s1", + ), expected) + + def test_host_mirror_override_preserves_other_registries(self): + template = self.env.from_string( + (Path(__file__).parents[1] / "roles/k3s/templates/registries.yaml.j2").read_text() + ) + mirrors = {"docker.io": ["docker.1ms.run", "docker.m.daocloud.io"], "ghcr.io": ["ghcr.m.daocloud.io"]} + for override in [{}, {"docker.io": list(reversed(mirrors["docker.io"]))}]: + result = yaml.safe_load(template.render( + registry_mirrors=mirrors, registry_mirror_overrides=override, + ))["mirrors"] + self.assertEqual(result["ghcr.io"]["endpoint"], ["https://ghcr.m.daocloud.io"]) + self.assertEqual(result["docker.io"]["endpoint"], [ + "https://" + host for host in override.get("docker.io", mirrors["docker.io"]) + ]) + if __name__ == "__main__": unittest.main()