feat(crowdsec): 完善配置支持nginx json日志格式并通过loki获取日志

2024-09-02 11:14:27 +08:00
@@ -12,29 +12,33 @@ spec:
    container_runtime: containerd
    image:
      tag: v1.6.1-2
-    config:
-      config.yaml.local: |
-        db_config:
-          type: postgresql
-          host: cnpg-cluster-hk-rw.infra-data
-          port: 5432
-          db_name: crowdsec
-          user: app
-          password: nyrHzh9WWlDZzvVw7bDFo74gKb9zsls0Sy7OwRTDWiRTNPQQQkW85taUFAoX2AIC
-          sslmode: require
    agent:
+      # 由于dataScope为loki，所以此处强制要求部署在loki所在的节点 以节省网络资源
      nodeSelector:
-        svccontroller.k3s.cattle.io/enablelb: "true"
-        # 由于使用了loki搜集日志 此处不再需要 所以使用此标签 禁用所有的agent
-        crowdsec/enabled: "true"
+        kubernetes.io/hostname: alihka 
+      # 此处无实际作用 只是为了过helmChart的校验
      acquisition:
        - namespace: infra-net
          podName: ingress-nginx-controller-*
          program: nginx
          poll_without_inotify: true
+      additionalAcquisition:
+        - source: loki
+          log_level: info
+          url: http://loki.infra-monitor:3100/
+          limit: 1000
+          query: |
+            {job="infra-net/ingress-nginx"}
+          labels:
+            type: nginx         
      env:
        - name: COLLECTIONS
-          value: "crowdsecurity/nginx"
+          value: "crowdsecurity/base-http-scenarios"
+        - name: SCENARIOS
+          value: "crowdsecurity/nginx-req-limit-exceeded"
+      persistentVolume:
+        config:
+          enabled: false
    lapi:
      affinity:
        nodeAffinity:
@@ -57,4 +61,95 @@ spec:
          enabled: false
        data:
          enabled: false
+    config:
+      config.yaml.local: |
+        db_config:
+          type: postgresql
+          host: cnpg-cluster-hk-rw.infra-data
+          port: 5432
+          db_name: crowdsec
+          user: app
+          password: nyrHzh9WWlDZzvVw7bDFo74gKb9zsls0Sy7OwRTDWiRTNPQQQkW85taUFAoX2AIC
+          sslmode: require
+      parsers:
+        s01-parse:
+          # 新增nginx json日志解析
+          nginx-logs.yaml: |
+            filter: "evt.Parsed.program startsWith 'nginx'"
+            onsuccess: next_stage
+            name: crowdsecurity/nginx-logs
+            description: "Parse nginx access and error logs"
+            pattern_syntax:
+              NGCUSTOMURIPATH: "(?:/[A-Za-z0-9$.+!*'\\(\\)\\{\\},~:;=@\\#%&_\\-]*)+"
+              NGCUSTOMURIPATHPARAM: '%{NGCUSTOMURIPATH}(?:%{URIPARAM})?'
+            nodes:
+              # nginx access logs
+              - filter: TrimSpace(evt.Parsed.message) startsWith "{" && UnmarshalJSON(evt.Parsed.message, evt.Unmarshaled, "nginx") in ["", nil]
+                statics:
+                  - meta: log_type
+                    value: http_access-log
+                  - target: evt.StrTime
+                    expression: evt.Parsed.time
+                  - meta: service
+                    value: http
+                  - meta: source_ip
+                    expression: evt.Unmarshaled.nginx.remote_addr
+                  - meta: http_status
+                    expression: evt.Unmarshaled.nginx.status
+                  - meta: http_path
+                    expression: evt.Unmarshaled.nginx.request
+                  - meta: http_user_agent
+                    expression: evt.Unmarshaled.nginx.http_user_agent
+              # nginx error logs
+              - grok:
+                  pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{NGCUSTOMURIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'
+                  apply_on: message
+                  statics:
+                    - meta: log_type
+                      value: http_error-log
+                    - target: evt.StrTime
+                      expression: evt.Parsed.time
+                    - meta: service
+                      value: http
+                    - meta: source_ip
+                      expression: "evt.Parsed.remote_addr"
+                    - meta: http_status
+                      expression: "evt.Parsed.status"
+                    - meta: http_path
+                      expression: "evt.Parsed.request"
+                    - meta: http_user_agent
+                      expression: "evt.Parsed.http_user_agent"
+                    - meta: target_fqdn
+                      expression: "evt.Parsed.target_fqdn"
+                pattern_syntax:
+                  NO_DOUBLE_QUOTE: '[^"]+'
+                onsuccess: next_stage
+                nodes:
+                  - filter: "evt.Parsed.message contains 'was not found in'"
+                    pattern_syntax:
+                      USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'
+                    grok:
+                      pattern: '%{USER_NOT_FOUND}'
+                      apply_on: message
+                    statics:
+                      - meta: sub_type
+                        value: "auth_fail"
+                      - meta: username
+                        expression: evt.Parsed.username
+                  - filter: "evt.Parsed.message contains 'password mismatch'"
+                    pattern_syntax:
+                      PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'
+                    grok:
+                      pattern: '%{PASSWORD_MISMATCH}'
+                      apply_on: message
+                    statics:
+                      - meta: sub_type
+                        value: "auth_fail"
+                      - meta: username
+                        expression: evt.Parsed.username
+                  - filter: "evt.Parsed.message contains 'limiting requests, excess'"
+                    statics:
+                      - meta: sub_type
+                        value: "req_limit_exceeded"
+