diff --git a/README.md b/README.md index 1ec48db..f01bc15 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ ### k3s 部署仓库 让你快速拥有一个高可用的k3s集群 并且具有完备的生产级能力(监控、告警、防护、负载、备份) -#### install 集群安装相关 +#### 集群安装相关 -参见 [install/README.md](install/README_MANUAL.md) +参见 [ansible/README.md](ansible/README.md) #### apps 相关应用 diff --git a/ansible/README.md b/ansible/README.md index c426a2f..a686e03 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -7,8 +7,6 @@ ``` ansible/ ├── ansible.cfg # Ansible 配置 -├── .ansible-lint # Lint 规则配置 -├── requirements.yml # Ansible Galaxy 依赖 ├── inventory/ │ ├── hosts.yml # 主机清单 ⭐ 需修改 │ └── group_vars/all.yml # 全局变量 diff --git a/ansible/roles/k3s/tasks/main.yml b/ansible/roles/k3s/tasks/main.yml index 0b41854..5d281ed 100644 --- a/ansible/roles/k3s/tasks/main.yml +++ b/ansible/roles/k3s/tasks/main.yml @@ -11,13 +11,20 @@ state: directory mode: "0755" -# 部署配置文件 +# 检查安装状态 +- name: Check if K3s is installed + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_binary + +# 部署配置文件（注册变更状态） - name: Deploy K3s server config ansible.builtin.template: src: k3s-server.yaml.j2 dest: /etc/rancher/k3s/config.yaml mode: "0600" when: "'masters' in group_names" + register: k3s_server_config - name: Deploy K3s agent config ansible.builtin.template: @@ -25,6 +32,7 @@ dest: /etc/rancher/k3s/config.yaml mode: "0600" when: "'agents' in group_names" + register: k3s_agent_config - name: Deploy registries.yaml ansible.builtin.template: @@ -33,17 +41,17 @@ mode: "0644" when: use_mirror | default(false) +# 判断是否需要安装/重启 +- name: Set K3s installation flag + ansible.builtin.set_fact: + k3s_needs_install: "{{ not k3s_binary.stat.exists or (k3s_server_config.changed | default(false)) or (k3s_agent_config.changed | default(false)) }}" + # 设置安装变量 - name: Set K3s install variables ansible.builtin.set_fact: k3s_install_url: "{{ mirror_k3s_install_url if (use_mirror | default(false)) else global_k3s_install_url }}" k3s_install_mirror: "{{ 'INSTALL_K3S_MIRROR=cn' if (use_mirror | default(false)) else '' }}" -# 检查安装状态 -- name: Check if K3s is installed - ansible.builtin.stat: - path: /usr/local/bin/k3s - register: k3s_binary # 下载安装脚本 - name: Download K3s install script @@ -51,7 +59,7 @@ url: "{{ k3s_install_url }}" dest: /tmp/k3s-install.sh mode: "0755" - when: not k3s_binary.stat.exists + when: k3s_needs_install # 安装 K3s - name: Install K3s server @@ -62,7 +70,7 @@ INSTALL_K3S_MIRROR: "{{ 'cn' if (use_mirror | default(false)) else '' }}" when: - "'masters' in group_names" - - not k3s_binary.stat.exists + - k3s_needs_install changed_when: true - name: Install K3s agent @@ -73,7 +81,7 @@ INSTALL_K3S_MIRROR: "{{ 'cn' if (use_mirror | default(false)) else '' }}" when: - "'agents' in group_names" - - not k3s_binary.stat.exists + - k3s_needs_install changed_when: true # 清理安装脚本 diff --git a/apps/README.md b/apps/README.md index 6fa4c4c..48d8ed0 100644 --- a/apps/README.md +++ b/apps/README.md @@ -1,46 +1,6 @@ ### apps -应用部署方法 - -```shell -kubectl apply -f apps/xxx -R -``` - -举例: - -```shell -kubectl apply -f apps/infra/data/redis -R -``` - -你可以一次性将所有的应用部署到k8s集群中 但是此处建议分开部署 每个文件夹单独执行 以保证不会出现错误与性能问题 - -注意!! 在部署前你需要替换yaml中的YOU_SHOULD_MODIFY_THIS_ 开头的字段 替换为自己的值 这些值的来源部分是自己生成的、部分是需要你自己去申请的 - -比如说你需要去华为云申请一个access key id和secret key 还有一个bucket name 这些值需要你自己去申请 - -### 应用说明 - -./kube文件夹下的请全部执行 此文件架内部为集群优化相关内容 例如dns延迟优化 -(patch-affinity.yaml 按需 仅在你想让k3s自带的system服务使用特定节点时使用 比如保留核心服务停留在高可用节点上) - -- infra-net: 网络相关的应用 - - nginx: 负载均衡服务 替换集群默认的ingress(traefik) - - crowdsec: 安全防护服务 - - tailscale: 集群内网加速服务 如果对集群内网加速没有需求 可以不安装 -- infra-data: 数据存储相关的应用 - - redis: redis服务 - - postgresql-ha: postgresql服务 - - cloudnative: postgresql服务 操作符版本 推荐 -- infra-devops: devops相关的应用 - - gitea: git托管服务 - - cert-manager: 证书管理服务 - - reflector: 密钥同步服务 - - velero: 备份服务 -- infra-monitor: 监控相关的应用 - - prometheus: 监控服务 - - loki: 日志服务 -- apps: 其他应用 个人应用部分 - - whoami: 测试服务 +集群服务helm部署的应用，包含一些基础服务和一些业务服务 ### 调试集群内服务方法 运行此命令 @@ -57,14 +17,13 @@ kubectl run -i --tty --rm --restart=Never \ 然后使用reflector将secret中的密钥同步到其他namespace中 ```shell -kubectl -n infra-devops create secret generic s3-devcm-hw \ +kubectl -n infra-data create secret generic s3-devcm-hw \ --from-literal=ACCESS_KEY_ID=xxxxx \ --from-literal=ACCESS_SECRET_KEY=xxxxx -kubectl -n infra-devops annotate secret s3-devcm-hw \ +kubectl -n infra-data annotate secret s3-devcm-hw \ reflector.v1.k8s.emberstack.com/reflection-allowed=true \ - reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces=infra-data \ - reflector.v1.k8s.emberstack.com/reflection-auto-enabled=true \ - reflector.v1.k8s.emberstack.com/reflection-auto-namespace=infra-data --overwrite + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces=infra-devops,apps \ + reflector.v1.k8s.emberstack.com/reflection-auto-enabled=true --overwrite ``` diff --git a/apps/apps/halo/helmchart.yaml b/apps/apps/halo/helmchart.yaml index b033f21..73fb7dd 100644 --- a/apps/apps/halo/helmchart.yaml +++ b/apps/apps/halo/helmchart.yaml @@ -41,6 +41,9 @@ spec: pathType: Prefix podAnnotations: backup.velero.io/backup-volumes: halo-data + persistence: + annotations: + helm.sh/resource-policy: keep metrics: enabled: true mysql: @@ -52,8 +55,9 @@ spec: host: cnpg17-cluster-hk-rw.infra-data port: 5432 user: app - password: FybaFtf6NV5jnxhj5bOPpHbO6KypZeHiyiskgAWkM5nioW2j82HtCf6GnW9xVKjE + password: from-secret database: halo + existingSecret: cnpg17-cluster-hk-app haloUsername: rohow haloExternalUrl: https://dev.cm diff --git a/apps/infra/data/cloudnative-pg/helmchart.yaml b/apps/infra/data/cloudnative-pg/helmchart.yaml index eb21f3a..cdf1865 100644 --- a/apps/infra/data/cloudnative-pg/helmchart.yaml +++ b/apps/infra/data/cloudnative-pg/helmchart.yaml @@ -19,7 +19,7 @@ spec: values: - "cn-sh" tolerations: - - key: "node-role.kubernetes.io/master" + - key: "node-role.kubernetes.io/control-plane" operator: "Exists" effect: "NoSchedule" image: diff --git a/apps/infra/data/redis/helmchart-sh.yaml b/apps/infra/data/redis/helmchart-sh.yaml deleted file mode 100644 index d45f457..0000000 --- a/apps/infra/data/redis/helmchart-sh.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: redis-cluster-sh - namespace: infra-data -spec: - chart: oci://registry-1.docker.io/bitnamicharts/redis - targetNamespace: infra-data - version: 20.7.0 - valuesContent: |- - global: - redis: - password: ribiPwYQNU6GWxCYR0Nj - master: - nodeAffinityPreset: - type: soft - key: topology.kubernetes.io/region - values: - - cn-sh - replica: - replicaCount: 0 - nodeAffinityPreset: - type: soft - key: topology.kubernetes.io/region - values: - - cn-sh - - diff --git a/apps/infra/data/valkey-cluster/helmchart.yaml b/apps/infra/data/valkey-cluster/helmchart.yaml new file mode 100644 index 0000000..f6fdb78 --- /dev/null +++ b/apps/infra/data/valkey-cluster/helmchart.yaml @@ -0,0 +1,21 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: valkey-cluster-sh + namespace: infra-data +spec: + chart: oci://registry-1.docker.io/bitnamicharts/valkey-cluster + targetNamespace: infra-data + version: 3.0.23 + valuesContent: |- + image: + repository: bitnamilegacy/valkey-cluster + cluster: + nodes: 1 + replicas: 0 + valkey: + nodeAffinityPreset: + type: hard + key: topology.kubernetes.io/region + values: + - cn-sh \ No newline at end of file diff --git a/apps/infra/devops/cert-manager/clusterissuer-dnspod.yaml b/apps/infra/devops/cert-manager/clusterissuer-dnspod.yaml new file mode 100644 index 0000000..214cbe4 --- /dev/null +++ b/apps/infra/devops/cert-manager/clusterissuer-dnspod.yaml @@ -0,0 +1,26 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: cert-manager-webhook-dnspod + labels: + app: cert-manager-webhook-dnspod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@dev.cm + privateKeySecretRef: + name: cert-manager-webhook-dnspod-letsencrypt + solvers: + - dns01: + cnameStrategy: Follow + webhook: + groupName: cert.dev.cm + solverName: dnspod + config: + ttl: 600 + secretIdRef: + name: dnspod-secret + key: secretId + secretKeyRef: + name: dnspod-secret + key: secretKey \ No newline at end of file diff --git a/apps/infra/devops/cert-manager/helmchart-dnspod.yaml b/apps/infra/devops/cert-manager/helmchart-dnspod.yaml index 736a3a4..c7e2104 100644 --- a/apps/infra/devops/cert-manager/helmchart-dnspod.yaml +++ b/apps/infra/devops/cert-manager/helmchart-dnspod.yaml @@ -9,17 +9,6 @@ spec: targetNamespace: infra-devops version: 1.4.5 valuesContent: |- - namespace: infra-devops - certManager: - namespace: infra-devops - groupName: cert.dev.cm - clusterIssuer: - # 此处需在部署后修改clusterIssuer 添加在dns01下 - # cnameStrategy: Follow - staging: false - email: admin@dev.cm - secretId: AKIDzmKdvDSfonogKip55pIVR6h7ScjaBWcg - secretKey: zudDdtytkPr8HI9oKeniSxIRPCmCe0CD affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,7 +18,12 @@ spec: operator: In values: - "cn-sh" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" + image: + tag: "1.5.2" + namespace: infra-devops + certManager: + namespace: infra-devops + groupName: cert.dev.cm + # 此处关闭 选择手动创建 以支持cnameStrategy + clusterIssuer: + enabled: false diff --git a/apps/infra/devops/cert-manager/helmchart.yaml b/apps/infra/devops/cert-manager/helmchart.yaml index 79395a0..38bb9a5 100644 --- a/apps/infra/devops/cert-manager/helmchart.yaml +++ b/apps/infra/devops/cert-manager/helmchart.yaml @@ -1,5 +1,3 @@ -# 需要提前安装crds -# kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.18.2/cert-manager.crds.yaml apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -9,7 +7,7 @@ spec: repo: https://charts.jetstack.io chart: cert-manager targetNamespace: infra-devops - version: v1.19.2 + version: v1.19.3 valuesContent: |- affinity: nodeAffinity: @@ -20,10 +18,6 @@ spec: operator: In values: - "cn-sh" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" webhook: affinity: nodeAffinity: @@ -34,10 +28,6 @@ spec: operator: In values: - "cn-sh" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" cainjector: affinity: nodeAffinity: @@ -48,14 +38,13 @@ spec: operator: In values: - "cn-sh" - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" + crds: + enabled: true + keep: true # 在删除证书时同时删除secret enableCertificateOwnerRef: true prometheus: - enabled: true + enabled: false servicemonitor: enabled: true interval: 300s diff --git a/apps/infra/devops/reflector/helmchart.yaml b/apps/infra/devops/reflector/helmchart.yaml index 1d0df90..ec00d6b 100644 --- a/apps/infra/devops/reflector/helmchart.yaml +++ b/apps/infra/devops/reflector/helmchart.yaml @@ -18,11 +18,3 @@ spec: operator: In values: - "cn-sh" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - tce diff --git a/apps/infra/devops/velero/helmchart.yaml b/apps/infra/devops/velero/helmchart.yaml index 74d5121..01dc71b 100644 --- a/apps/infra/devops/velero/helmchart.yaml +++ b/apps/infra/devops/velero/helmchart.yaml @@ -25,11 +25,9 @@ spec: - key: kubernetes.io/hostname operator: In values: - - homea - kubectl: - image: - repository: alpine/k8s - tag: "1.34.0" + - homeb + # 此处暂时切换关闭upgradeCRDs操作 待官方修复后再开启 + upgradeCRDs: false deployNodeAgent: true snapshotsEnabled: false configuration: @@ -50,13 +48,19 @@ spec: s3ForcePathStyle: false s3Url: https://obs.cn-east-3.myhuaweicloud.com checksumAlgorithm: "" + extraEnvVars: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-devcm-hw + key: ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-devcm-hw + key: ACCESS_SECRET_KEY credentials: - useSecret: true - secretContents: - cloud: | - [default] - aws_access_key_id = A9RI5BC15F3L9EI8T51T - aws_secret_access_key = ky1n3OlNNu7wjgctVjCqb03HWxjZucRGhvcEBp51 + useSecret: false initContainers: - name: velero-plugin-for-aws image: velero/velero-plugin-for-aws:v1.13.0 diff --git a/apps/infra/gitops/gitea/helmchart.yaml b/apps/infra/gitops/gitea/helmchart.yaml index 11b9b09..a41fc7c 100644 --- a/apps/infra/gitops/gitea/helmchart.yaml +++ b/apps/infra/gitops/gitea/helmchart.yaml @@ -67,17 +67,13 @@ spec: HOST: cnpg17-cluster-sh-rw.infra-data:5432 NAME: gitea USER: app - PASSWD: HueUoQx05DM0ICBPu1GrmBvBXE6NO3poKE6yPqokPv3dPpWvWRLAr3RXSpaL3AZd SSL_MODE: disable session: PROVIDER: redis - PROVIDER_CONFIG: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0 cache: ADAPTER: redis - HOST: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0?pool_size=100&idle_timeout=180s queue: TYPE: redis - CONN_STR: redis://:ribiPwYQNU6GWxCYR0Nj@redis-cluster-sh-master.infra-data:6379/0 repository: DEFAULT_REPO_UNITS: repo.code,repo.releases,repo.issues,repo.pulls actions: @@ -99,6 +95,23 @@ spec: ui: THEMES: gitea-auto, gitea-light, gitea-dark, github-auto, github-light, github-dark, github-soft-dark DEFAULT_THEME: github-auto + additionalConfigFromEnvs: + - name: GITEA__DATABASE__PASSWD + valueFrom: + secretKeyRef: + name: cnpg17-cluster-sh-app + key: password + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-cluster-sh + key: valkey-password + - name: GITEA__SESSION__PROVIDER_CONFIG + value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s" + - name: GITEA__CACHE__HOST + value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s" + - name: GITEA__QUEUE__CONN_STR + value: "redis://:$(REDIS_PASSWORD)@valkey-cluster-sh-headless.infra-data:6379/0?pool_size=100&idle_timeout=180s" valkey-cluster: enabled: false extraVolumes: diff --git a/apps/infra/monitor/prometheus/helmchart.yaml b/apps/infra/monitor/prometheus/helmchart.yaml index 6580ab5..ea1fa4c 100644 --- a/apps/infra/monitor/prometheus/helmchart.yaml +++ b/apps/infra/monitor/prometheus/helmchart.yaml @@ -68,7 +68,7 @@ spec: host: cnpg17-cluster-sh-rw.infra-data:5432 name: grafana user: app - password: HueUoQx05DM0ICBPu1GrmBvBXE6NO3poKE6yPqokPv3dPpWvWRLAr3RXSpaL3AZd + password: fYyAc4PNKLrvEB0IfkDm1TMR7sZkAcK1DGp4yqG5Y9aSS0UJMCgSiW6hhrsTztLA persistence: type: pvc enabled: true diff --git a/apps/infra/net/crowdsec/helmchart.yaml b/apps/infra/net/crowdsec/helmchart.yaml index 6eefc22..5e00bde 100644 --- a/apps/infra/net/crowdsec/helmchart.yaml +++ b/apps/infra/net/crowdsec/helmchart.yaml @@ -102,7 +102,7 @@ spec: port: 5432 db_name: crowdsec user: app - password: FybaFtf6NV5jnxhj5bOPpHbO6KypZeHiyiskgAWkM5nioW2j82HtCf6GnW9xVKjE + password: 4EMiSg9adUSxPAwNWIsHhKd1WZ7lhGuCnNofCFHuU1aQHSho85xeSK6TPcgJ4NU7 sslmode: require api: server: diff --git a/apps/infra/net/nginx/helmchart.yaml b/apps/infra/net/nginx/helmchart.yaml index 02340a5..8413644 100644 --- a/apps/infra/net/nginx/helmchart.yaml +++ b/apps/infra/net/nginx/helmchart.yaml @@ -19,7 +19,7 @@ spec: nodeSelector: svccontroller.k3s.cattle.io/enablelb: "true" tolerations: - - key: "node-role.kubernetes.io/master" + - key: "node-role.kubernetes.io/control-plane" operator: "Exists" effect: "NoSchedule" labels: diff --git a/apps/kube/coredns/nodelocaldns.yaml b/apps/kube/coredns/nodelocaldns.yaml index 80c0ffa..8d4c88d 100644 --- a/apps/kube/coredns/nodelocaldns.yaml +++ b/apps/kube/coredns/nodelocaldns.yaml @@ -126,7 +126,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane operator: NotIn values: - "true" @@ -143,7 +143,7 @@ spec: operator: "Exists" containers: - name: node-cache - image: registry.k8s.io/dns/k8s-dns-node-cache:1.25.0 + image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.7 resources: requests: cpu: 25m diff --git a/apps/kube/patch-affinity.yaml b/apps/kube/patch-affinity.yaml index 7d6d312..c5722bc 100644 --- a/apps/kube/patch-affinity.yaml +++ b/apps/kube/patch-affinity.yaml @@ -6,14 +6,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane operator: In values: - - "true" - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule \ No newline at end of file + - "true" \ No newline at end of file