feat(k3s): 密码切换为密钥 & 相关服务升级

README.md

@@ -1,8 +1,8 @@
 ### k3s 部署仓库 让你快速拥有一个高可用的k3s集群 并且具有完备的生产级能力(监控、告警、防护、负载、备份)
 
-#### install 集群安装相关
+#### 集群安装相关
 
-参见 [install/README.md](install/README_MANUAL.md)
+参见 [ansible/README.md](ansible/README.md)
 
 #### apps 相关应用
 

ansible/README.md

@@ -7,8 +7,6 @@
 ```
 ansible/
 ├── ansible.cfg                 # Ansible 配置
-├── .ansible-lint               # Lint 规则配置
-├── requirements.yml            # Ansible Galaxy 依赖
 ├── inventory/
 │   ├── hosts.yml               # 主机清单 ⭐ 需修改
 │   └── group_vars/all.yml      # 全局变量

ansible/inventory/hosts.yml

@@ -56,7 +56,6 @@ all:
           ansible_host: tchk.node.dev.cm
           node_hostname: tchk
           node_region: cn-hk
-          enable_lb: true
         tthk:
           ansible_host: tthk.node.dev.cm
           node_hostname: tthk
@@ -83,6 +82,7 @@ all:
           ansible_host: hwsg.node.dev.cm
           node_hostname: hwsg
           node_region: sg-sg
+          enable_lb: true
           netfilter_mode: nodivert
         hwa:
           ansible_host: hwa.node.dev.cm

ansible/roles/k3s/tasks/main.yml

@@ -11,13 +11,20 @@
     state: directory
     mode: "0755"
 
-# 部署配置文件
+# 检查安装状态
+- name: Check if K3s is installed
+  ansible.builtin.stat:
+    path: /usr/local/bin/k3s
+  register: k3s_binary
+
+# 部署配置文件（注册变更状态）
 - name: Deploy K3s server config
   ansible.builtin.template:
     src: k3s-server.yaml.j2
     dest: /etc/rancher/k3s/config.yaml
     mode: "0600"
   when: "'masters' in group_names"
+  register: k3s_server_config
 
 - name: Deploy K3s agent config
   ansible.builtin.template:
@@ -25,6 +32,7 @@
     dest: /etc/rancher/k3s/config.yaml
     mode: "0600"
   when: "'agents' in group_names"
+  register: k3s_agent_config
 
 - name: Deploy registries.yaml
   ansible.builtin.template:
@@ -33,17 +41,17 @@
     mode: "0644"
   when: use_mirror | default(false)
 
+# 判断是否需要安装/重启
+- name: Set K3s installation flag
+  ansible.builtin.set_fact:
+    k3s_needs_install: "{{ not k3s_binary.stat.exists or (k3s_server_config.changed | default(false)) or (k3s_agent_config.changed | default(false)) }}"
+
 # 设置安装变量
 - name: Set K3s install variables
   ansible.builtin.set_fact:
     k3s_install_url: "{{ mirror_k3s_install_url if (use_mirror | default(false)) else global_k3s_install_url }}"
     k3s_install_mirror: "{{ 'INSTALL_K3S_MIRROR=cn' if (use_mirror | default(false)) else '' }}"
 
-# 检查安装状态
-- name: Check if K3s is installed
-  ansible.builtin.stat:
-    path: /usr/local/bin/k3s
-  register: k3s_binary
 
 # 下载安装脚本
 - name: Download K3s install script
@@ -51,7 +59,7 @@
     url: "{{ k3s_install_url }}"
     dest: /tmp/k3s-install.sh
     mode: "0755"
-  when: not k3s_binary.stat.exists
+  when: k3s_needs_install
 
 # 安装 K3s
 - name: Install K3s server
@@ -62,7 +70,7 @@
     INSTALL_K3S_MIRROR: "{{ 'cn' if (use_mirror | default(false)) else '' }}"
   when:
     - "'masters' in group_names"
-    - not k3s_binary.stat.exists
+    - k3s_needs_install
   changed_when: true
 
 - name: Install K3s agent
@@ -73,7 +81,7 @@
     INSTALL_K3S_MIRROR: "{{ 'cn' if (use_mirror | default(false)) else '' }}"
   when:
     - "'agents' in group_names"
-    - not k3s_binary.stat.exists
+    - k3s_needs_install
   changed_when: true
 
 # 清理安装脚本

apps/README.md

@@ -1,46 +1,6 @@
 ### apps
 
-应用部署方法
+集群服务helm部署的应用，包含一些基础服务和一些业务服务
-
-```shell
-kubectl apply -f apps/xxx -R
-```
-
-举例:
-
-```shell
-kubectl apply -f apps/infra/data/redis -R
-```
-
-你可以一次性将所有的应用部署到k8s集群中 但是此处建议分开部署 每个文件夹单独执行 以保证不会出现错误与性能问题
-
-注意!! 在部署前你需要替换yaml中的YOU_SHOULD_MODIFY_THIS_ 开头的字段 替换为自己的值 这些值的来源部分是自己生成的、部分是需要你自己去申请的
-
-比如说你需要去华为云申请一个access key id和secret key 还有一个bucket name 这些值需要你自己去申请
-
-### 应用说明
-
-./kube文件夹下的请全部执行 此文件架内部为集群优化相关内容 例如dns延迟优化
-(patch-affinity.yaml 按需 仅在你想让k3s自带的system服务使用特定节点时使用 比如保留核心服务停留在高可用节点上)
-
-- infra-net: 网络相关的应用
-    - nginx: 负载均衡服务 替换集群默认的ingress(traefik)
-    - crowdsec: 安全防护服务
-    - tailscale: 集群内网加速服务 如果对集群内网加速没有需求 可以不安装
-- infra-data: 数据存储相关的应用
-    - redis: redis服务
-    - postgresql-ha: postgresql服务
-    - cloudnative: postgresql服务 操作符版本 推荐
-- infra-devops: devops相关的应用
-    - gitea: git托管服务
-    - cert-manager: 证书管理服务
-    - reflector: 密钥同步服务
-    - velero: 备份服务
-- infra-monitor: 监控相关的应用
-    - prometheus: 监控服务
-    - loki: 日志服务
-- apps: 其他应用 个人应用部分
-    - whoami: 测试服务
 
 ### 调试集群内服务方法 运行此命令
 
@@ -57,14 +17,13 @@ kubectl run -i --tty --rm --restart=Never \
 然后使用reflector将secret中的密钥同步到其他namespace中
 
 ```shell
-kubectl -n infra-devops create secret generic s3-devcm-hw \
+kubectl -n infra-data create secret generic s3-devcm-hw \
     --from-literal=ACCESS_KEY_ID=xxxxx \
     --from-literal=ACCESS_SECRET_KEY=xxxxx
     
-kubectl -n infra-devops annotate secret s3-devcm-hw \
+kubectl -n infra-data annotate secret s3-devcm-hw \
     reflector.v1.k8s.emberstack.com/reflection-allowed=true \
-    reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces=infra-data \
+    reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces=infra-devops,apps \
-    reflector.v1.k8s.emberstack.com/reflection-auto-enabled=true \
+    reflector.v1.k8s.emberstack.com/reflection-auto-enabled=true --overwrite
-    reflector.v1.k8s.emberstack.com/reflection-auto-namespace=infra-data --overwrite
     
 ```

apps/apps/halo/helmchart.yaml

@@ -41,6 +41,9 @@ spec:
       pathType: Prefix
     podAnnotations:
       backup.velero.io/backup-volumes: halo-data
+    persistence:
+      annotations:
+        helm.sh/resource-policy: keep
     metrics:
       enabled: true
     mysql:
@@ -52,8 +55,9 @@ spec:
       host: cnpg17-cluster-hk-rw.infra-data
       port: 5432
       user: app
-      password: FybaFtf6NV5jnxhj5bOPpHbO6KypZeHiyiskgAWkM5nioW2j82HtCf6GnW9xVKjE
+      password: from-secret
       database: halo
+      existingSecret: cnpg17-cluster-hk-app
     haloUsername: rohow
     haloExternalUrl: https://dev.cm
 

apps/infra/data/cloudnative-pg/helmchart.yaml

@@ -19,7 +19,7 @@ spec:
                   values:
                     - "cn-sh"
     tolerations:
-      - key: "node-role.kubernetes.io/master"
+      - key: "node-role.kubernetes.io/control-plane"
         operator: "Exists"
         effect: "NoSchedule"
     image:

apps/infra/data/redis/helmchart-sh.yaml

@@ -1,28 +0,0 @@
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: redis-cluster-sh
-  namespace: infra-data
-spec:
-  chart: oci://registry-1.docker.io/bitnamicharts/redis
-  targetNamespace: infra-data
-  version: 20.7.0
-  valuesContent: |-
-    global:
-      redis:
-        password: ribiPwYQNU6GWxCYR0Nj
-    master:
-      nodeAffinityPreset:
-        type: soft
-        key: topology.kubernetes.io/region
-        values:
-          - cn-sh
-    replica:
-      replicaCount: 0
-      nodeAffinityPreset:
-        type: soft
-        key: topology.kubernetes.io/region
-        values:
-          - cn-sh   
-
-

apps/infra/data/valkey-cluster/helmchart.yaml

@@ -0,0 +1,21 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: valkey-cluster-sh
+  namespace: infra-data
+spec:
+  chart: oci://registry-1.docker.io/bitnamicharts/valkey-cluster
+  targetNamespace: infra-data
+  version: 3.0.23
+  valuesContent: |-
+    image:
+      repository: bitnamilegacy/valkey-cluster
+    cluster:
+      nodes: 1
+      replicas: 0
+    valkey: 
+      nodeAffinityPreset:
+        type: hard
+        key: topology.kubernetes.io/region
+        values:
+          - cn-sh

apps/infra/devops/cert-manager/clusterissuer-dnspod.yaml

@@ -0,0 +1,26 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: cert-manager-webhook-dnspod
+  labels:
+    app: cert-manager-webhook-dnspod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: admin@dev.cm
+    privateKeySecretRef:
+      name: cert-manager-webhook-dnspod-letsencrypt
+    solvers:
+      - dns01:
+          cnameStrategy: Follow
+          webhook:
+            groupName: cert.dev.cm
+            solverName: dnspod
+            config:
+              ttl: 600
+              secretIdRef:
+                name: dnspod-secret
+                key: secretId
+              secretKeyRef:
+                name: dnspod-secret
+                key: secretKey

apps/infra/devops/cert-manager/helmchart-dnspod.yaml

@@ -9,17 +9,6 @@ spec:
   targetNamespace: infra-devops
   version: 1.4.5
   valuesContent: |-
-    namespace: infra-devops
-    certManager:
-      namespace: infra-devops
-    groupName: cert.dev.cm
-    clusterIssuer:
-      # 此处需在部署后修改clusterIssuer 添加在dns01下
-      # cnameStrategy: Follow
-      staging: false
-      email: admin@dev.cm
-      secretId: AKIDzmKdvDSfonogKip55pIVR6h7ScjaBWcg
-      secretKey: zudDdtytkPr8HI9oKeniSxIRPCmCe0CD
     affinity:
       nodeAffinity:
         requiredDuringSchedulingIgnoredDuringExecution:
@@ -29,7 +18,12 @@ spec:
                   operator: In
                   values:
                     - "cn-sh"
-    tolerations:
+    image:
-      - key: "node-role.kubernetes.io/master"
+      tag: "1.5.2"
-        operator: "Exists"
+    namespace: infra-devops
-        effect: "NoSchedule"
+    certManager:
+      namespace: infra-devops
+    groupName: cert.dev.cm
+    # 此处关闭 选择手动创建 以支持cnameStrategy
+    clusterIssuer:
+      enabled: false

apps/infra/devops/cert-manager/helmchart.yaml

@@ -1,5 +1,3 @@
-# 需要提前安装crds
-# kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.crds.yaml
 apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
@@ -9,7 +7,7 @@ spec:
   repo: https://charts.jetstack.io
   chart: cert-manager
   targetNamespace: infra-devops
-  version: v1.19.2
+  version: v1.19.3
   valuesContent: |-
     affinity:
       nodeAffinity:
@@ -20,10 +18,6 @@ spec:
                   operator: In
                   values:
                     - "cn-sh"
-    tolerations:
-      - key: "node-role.kubernetes.io/master"
-        operator: "Exists"
-        effect: "NoSchedule"
     webhook:
       affinity:
         nodeAffinity:
@@ -34,10 +28,6 @@ spec:
                     operator: In
                     values:
                       - "cn-sh"
-      tolerations:
-        - key: "node-role.kubernetes.io/master"
-          operator: "Exists"
-          effect: "NoSchedule"
     cainjector:
       affinity:
         nodeAffinity:
@@ -48,14 +38,13 @@ spec:
                     operator: In
                     values:
                       - "cn-sh"
-      tolerations:
+    crds:
-        - key: "node-role.kubernetes.io/master"
+      enabled: true
-          operator: "Exists"
+      keep: true
-          effect: "NoSchedule"
     # 在删除证书时同时删除secret
     enableCertificateOwnerRef: true
     prometheus:
-      enabled: true
+      enabled: false
       servicemonitor:
         enabled: true
         interval: 300s

apps/infra/devops/reflector/helmchart.yaml

@@ -18,11 +18,3 @@ spec:
                   operator: In
                   values:
                     - "cn-sh"
-        preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 1
-            preference:
-              matchExpressions:
-                - key: kubernetes.io/hostname
-                  operator: In
-                  values:
-                    - tce

apps/infra/devops/velero/helmchart.yaml

@@ -25,11 +25,9 @@ spec:
                 - key: kubernetes.io/hostname
                   operator: In
                   values:
-                    - homea  
+                    - homeb  
-    kubectl:
+    # 此处暂时切换关闭upgradeCRDs操作 待官方修复后再开启
-      image:
+    upgradeCRDs: false
-        repository: alpine/k8s
-        tag: "1.34.0"
     deployNodeAgent: true
     snapshotsEnabled: false
     configuration:
@@ -50,13 +48,19 @@ spec:
             s3ForcePathStyle: false
             s3Url: https://obs.cn-east-3.myhuaweicloud.com
             checksumAlgorithm: ""
+      extraEnvVars:
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: s3-devcm-hw
+              key: ACCESS_KEY_ID    
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: s3-devcm-hw
+              key: ACCESS_SECRET_KEY
     credentials:
-      useSecret: true
+      useSecret: false
-      secretContents:
-        cloud: |
-          [default]
-          aws_access_key_id = A9RI5BC15F3L9EI8T51T
-          aws_secret_access_key = ky1n3OlNNu7wjgctVjCqb03HWxjZucRGhvcEBp51
     initContainers:
       - name: velero-plugin-for-aws
         image: velero/velero-plugin-for-aws:v1.13.0

apps/infra/gitops/gitea/helmchart.yaml

@@ -67,17 +67,13 @@ spec:
           HOST: cnpg17-cluster-sh-rw.infra-data:5432
           NAME: gitea
           USER: app
-          PASSWD: HueUoQx05DM0ICBPu1GrmBvBXE6NO3poKE6yPqokPv3dPpWvWRLAr3RXSpaL3AZd
           SSL_MODE: disable
         session:
           PROVIDER: redis
-          PROVIDER_CONFIG: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0
         cache:
           ADAPTER: redis
-          HOST: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0?pool_size=100&idle_timeout=180s
         queue:
           TYPE: redis
-          CONN_STR: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0
         repository:
           DEFAULT_REPO_UNITS: repo.code,repo.releases,repo.issues,repo.pulls
         actions:
@@ -99,6 +95,23 @@ spec:
         ui:
           THEMES: gitea-auto, gitea-light, gitea-dark, github-auto, github-light, github-dark, github-soft-dark
           DEFAULT_THEME: github-auto
+      additionalConfigFromEnvs:
+        - name: GITEA__DATABASE__PASSWD
+          valueFrom:
+            secretKeyRef:
+              name: cnpg17-cluster-sh-app
+              key: password
+        - name: REDIS_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: valkey-cluster-sh
+              key: valkey-password
+        - name: GITEA__SESSION__PROVIDER_CONFIG
+          value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s"
+        - name: GITEA__CACHE__HOST
+          value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s"
+        - name: GITEA__QUEUE__CONN_STR
+          value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s"    
     valkey-cluster:
       enabled: false
     extraVolumes:

apps/infra/monitor/prometheus/helmchart.yaml

@@ -54,6 +54,11 @@ spec:
         paths:
           - /
       assertNoLeakedSecrets: false
+      envValueFrom:
+        GF_DATABASE_PASSWORD:
+          secretKeyRef:
+            name: cnpg17-cluster-sh-app
+            key: password
       grafana.ini:
         server:
           root_url: https://monitor.dev.cm/
@@ -68,7 +73,7 @@ spec:
           host: cnpg17-cluster-sh-rw.infra-data:5432
           name: grafana
           user: app
-          password: HueUoQx05DM0ICBPu1GrmBvBXE6NO3poKE6yPqokPv3dPpWvWRLAr3RXSpaL3AZd
+          password: $__env{GF_DATABASE_PASSWORD}
       persistence:
         type: pvc
         enabled: true

apps/infra/net/crowdsec/helmchart.yaml

@@ -93,6 +93,12 @@ spec:
           enabled: false
         data:
           enabled: false
+      env:
+        - name: DB_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: cnpg17-cluster-hk-app
+              key: password
     config:
       # api config.yaml配置
       config.yaml.local: |
@@ -102,7 +108,7 @@ spec:
           port: 5432
           db_name: crowdsec
           user: app
-          password: FybaFtf6NV5jnxhj5bOPpHbO6KypZeHiyiskgAWkM5nioW2j82HtCf6GnW9xVKjE
+          password: ${DB_PASSWORD}
           sslmode: require
         api:
           server:

apps/infra/net/nginx/helmchart.yaml

@@ -19,7 +19,7 @@ spec:
       nodeSelector:
         svccontroller.k3s.cattle.io/enablelb: "true"
       tolerations:
-        - key: "node-role.kubernetes.io/master"
+        - key: "node-role.kubernetes.io/control-plane"
           operator: "Exists"
           effect: "NoSchedule"
       labels:

apps/infra/net/tailscale/helmchart.yaml

@@ -12,7 +12,7 @@ spec:
     nodeSelector:
       kubernetes.io/hostname: tchk
     image:
-      tag: v1.90.8
+      tag: v1.94.1
     hostNetwork: true
     extraVolumes:
       - name: cert-volume

apps/kube/coredns/nodelocaldns.yaml

@@ -126,7 +126,7 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-                  - key: node-role.kubernetes.io/master
+                  - key: node-role.kubernetes.io/control-plane
                     operator: NotIn
                     values:
                       - "true"
@@ -143,7 +143,7 @@ spec:
           operator: "Exists"
       containers:
         - name: node-cache
-          image: registry.k8s.io/dns/k8s-dns-node-cache:1.25.0
+          image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.7
           resources:
             requests:
               cpu: 25m

apps/kube/patch-affinity.yaml

@@ -6,14 +6,7 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-                  - key: node-role.kubernetes.io/master
+                  - key: node-role.kubernetes.io/control-plane
                     operator: In
                     values:
-                      - "true"
+                      - "true"
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule