From 23723c03cf8e8df78838d60df5e876be38416538 Mon Sep 17 00:00:00 2001 From: rohow Date: Mon, 2 Sep 2024 11:14:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(crowdsec):=20=E5=AE=8C=E5=96=84=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20=E6=94=AF=E6=8C=81nginx=20json=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=20=E5=B9=B6=E9=80=9A=E8=BF=87loki=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/infra/monitor/crowdsec/helmchart.yaml | 123 ++++++++++++++++++--- 1 file changed, 109 insertions(+), 14 deletions(-) diff --git a/apps/infra/monitor/crowdsec/helmchart.yaml b/apps/infra/monitor/crowdsec/helmchart.yaml index 06998ad..808d608 100644 --- a/apps/infra/monitor/crowdsec/helmchart.yaml +++ b/apps/infra/monitor/crowdsec/helmchart.yaml @@ -12,29 +12,33 @@ spec: container_runtime: containerd image: tag: v1.6.1-2 - config: - config.yaml.local: | - db_config: - type: postgresql - host: cnpg-cluster-hk-rw.infra-data - port: 5432 - db_name: crowdsec - user: app - password: nyrHzh9WWlDZzvVw7bDFo74gKb9zsls0Sy7OwRTDWiRTNPQQQkW85taUFAoX2AIC - sslmode: require agent: + # 由于dataScope为loki，所以此处强制要求部署在loki所在的节点 以节省网络资源 nodeSelector: - svccontroller.k3s.cattle.io/enablelb: "true" - # 由于使用了loki搜集日志 此处不再需要 所以使用此标签 禁用所有的agent - crowdsec/enabled: "true" + kubernetes.io/hostname: alihka + # 此处无实际作用 只是为了过helmChart的校验 acquisition: - namespace: infra-net podName: ingress-nginx-controller-* program: nginx poll_without_inotify: true + additionalAcquisition: + - source: loki + log_level: info + url: http://loki.infra-monitor:3100/ + limit: 1000 + query: | + {job="infra-net/ingress-nginx"} + labels: + type: nginx env: - name: COLLECTIONS - value: "crowdsecurity/nginx" + value: "crowdsecurity/base-http-scenarios" + - name: SCENARIOS + value: "crowdsecurity/nginx-req-limit-exceeded" + persistentVolume: + config: + enabled: false lapi: affinity: nodeAffinity: @@ -57,4 +61,95 @@ spec: enabled: false data: enabled: false + config: + config.yaml.local: | + db_config: + type: postgresql + host: cnpg-cluster-hk-rw.infra-data + port: 5432 + db_name: crowdsec + user: app + password: nyrHzh9WWlDZzvVw7bDFo74gKb9zsls0Sy7OwRTDWiRTNPQQQkW85taUFAoX2AIC + sslmode: require + parsers: + s01-parse: + # 新增nginx json日志解析 + nginx-logs.yaml: | + filter: "evt.Parsed.program startsWith 'nginx'" + onsuccess: next_stage + name: crowdsecurity/nginx-logs + description: "Parse nginx access and error logs" + pattern_syntax: + NGCUSTOMURIPATH: "(?:/[A-Za-z0-9$.+!*'\\(\\)\\{\\},~:;=@\\#%&_\\-]*)+" + NGCUSTOMURIPATHPARAM: '%{NGCUSTOMURIPATH}(?:%{URIPARAM})?' + nodes: + # nginx access logs + - filter: TrimSpace(evt.Parsed.message) startsWith "{" && UnmarshalJSON(evt.Parsed.message, evt.Unmarshaled, "nginx") in ["", nil] + statics: + - meta: log_type + value: http_access-log + - target: evt.StrTime + expression: evt.Parsed.time + - meta: service + value: http + - meta: source_ip + expression: evt.Unmarshaled.nginx.remote_addr + - meta: http_status + expression: evt.Unmarshaled.nginx.status + - meta: http_path + expression: evt.Unmarshaled.nginx.request + - meta: http_user_agent + expression: evt.Unmarshaled.nginx.http_user_agent + # nginx error logs + - grok: + pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{NGCUSTOMURIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"' + apply_on: message + statics: + - meta: log_type + value: http_error-log + - target: evt.StrTime + expression: evt.Parsed.time + - meta: service + value: http + - meta: source_ip + expression: "evt.Parsed.remote_addr" + - meta: http_status + expression: "evt.Parsed.status" + - meta: http_path + expression: "evt.Parsed.request" + - meta: http_user_agent + expression: "evt.Parsed.http_user_agent" + - meta: target_fqdn + expression: "evt.Parsed.target_fqdn" + pattern_syntax: + NO_DOUBLE_QUOTE: '[^"]+' + onsuccess: next_stage + nodes: + - filter: "evt.Parsed.message contains 'was not found in'" + pattern_syntax: + USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"' + grok: + pattern: '%{USER_NOT_FOUND}' + apply_on: message + statics: + - meta: sub_type + value: "auth_fail" + - meta: username + expression: evt.Parsed.username + - filter: "evt.Parsed.message contains 'password mismatch'" + pattern_syntax: + PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch' + grok: + pattern: '%{PASSWORD_MISMATCH}' + apply_on: message + statics: + - meta: sub_type + value: "auth_fail" + - meta: username + expression: evt.Parsed.username + - filter: "evt.Parsed.message contains 'limiting requests, excess'" + statics: + - meta: sub_type + value: "req_limit_exceeded" +