Nginx Question posted in
The official documentation can be found here.

Nested JSON messages are not getting parsed correctly through logstash – Nginx

I am trying to parse modsecurity audit logs which are natively being logged in JSON format. However when I am using logstash to ingest in elastic stack those appears like below and actual needed fields are not being parsed in their respective fields. I am not sure if any further config needed in logstash?

Here is my logstash config

input {
      file {
        type => "json"
        path => "/var/log/modsec_audit.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
      }
    }
filter {
        json {
          source => "message"
                skip_on_invalid_json => "true"
          tag_on_failure => ["_jsonparsefailure"]
        }
      }

However in Kibana those are not getting parsed correctly.
Kibana Images for Logs Parsed

enter image description here

{
  "_index": "applox-2021.05.06",
  "_type": "_doc",
  "_id": "gldWQXkBPJxogvJE-ilc",
  "_score": 1,
  "_source": {
    "host": "cwaf",
    "type": "json",
    "@version": "1",
    "path": "/var/log/modsec_audit.log",
    "transaction": {
      "host_port": 80,
      "unique_id": "16202916142.116840",
      "messages": [
        {
          "message": "Method is not allowed by policy",
          "details": {
            "reference": "v0,6",
            "maturity": "9",
            "tags": [
              "application-multi",
              "language-multi",
              "platform-multi",
              "attack-generic",
              "OWASP_CRS/POLICY/METHOD_NOT_ALLOWED",
              "WASCTC/WASC-15",
              "OWASP_TOP_10/A6",
              "OWASP_AppSensor/RE1",
              "PCI/12.1"
            ],
            "ruleId": "911100",
            "file": "/etc/nginx/modsec/crs/owaspcrs/rules/REQUEST-911-METHOD-ENFORCEMENT.conf",
            "rev": "2",
            "lineNumber": "27",
            "ver": "OWASP_CRS/3.0.0",
            "accuracy": "9",
            "severity": "2",
            "match": "Matched "Operator `Within' with parameter `GET HEAD POST OPTIONS' against variable `REQUEST_METHOD' (Value: `DELETE' )",
            "data": "DELETE"
          }
        },
        {
          "message": "Inbound Anomaly Score Exceeded (Total Score: 1000)",
          "details": {
            "reference": "",
            "maturity": "0",
            "tags": [
              "application-multi",
              "language-multi",
              "platform-multi",
              "attack-generic"
            ],
            "ruleId": "949110",
            "file": "/etc/nginx/modsec/crs/owaspcrs/rules/REQUEST-949-BLOCKING-EVALUATION.conf",
            "rev": "",
            "lineNumber": "44",
            "ver": "",
            "accuracy": "0",
            "severity": "2",
            "match": "Matched "Operator `Ge' with parameter `5' against variable `TX:ANOMALY_SCORE' (Value: `1000' )",
            "data": ""
          }
        }
      ],
      "response": {
        "http_code": 403,
        "body": "<html>rn<head><title>403 Forbidden</title></head>rn<body>rn<center><h1>403 Forbidden</h1></center>rn<hr><center>nginx/1.20.0</center>rn</body>rn</html>rn",
        "headers": {
          "Server": "nginx/1.20.0",
          "Connection": "keep-alive",
          "Content-Length": "153",
          "Date": "Thu, 06 May 2021 09:00:14 GMT",
          "Content-Type": "text/html"
        }
      },
      "client_ip": "127.0.0.1",
      "host_ip": "127.0.0.1",
      "server_id": "30ef83bdf67cbb820de6ae6410fba661c0a34ea0",
      "request": {
        "uri": "/test.php",
        "method": "DELETE",
        "http_version": 1.1,
        "headers": {
          "Accept": "*/*",
          "Host": "localhost",
          "User-Agent": "curl/7.58.0"
        }
      },
      "producer": {
        "secrules_engine": "Enabled",
        "modsecurity": "ModSecurity v3.0.4 (Linux)",
        "components": [
          "OWASP_CRS/3.0.2""
        ],
        "connector": "ModSecurity-nginx v1.0.1"
      },
      "client_port": 39504,
      "time_stamp": "Thu May  6 14:30:14 2021"
    },
    "message": "{"transaction":{"client_ip":"127.0.0.1","time_stamp":"Thu May  6 14:30:14 2021","server_id":"30ef83bdf67cbb820de6ae6410fba661c0a34ea0","client_port":39504,"host_ip":"127.0.0.1","host_port":80,"unique_id":"16202916142.116840","request":{"method":"DELETE","http_version":1.1,"uri":"/test.php","headers":{"Host":"localhost","User-Agent":"curl/7.58.0","Accept":"*/*"}},"response":{"body":"<html>\r\n<head><title>403 Forbidden</title></head>\r\n<body>\r\n<center><h1>403 Forbidden</h1></center>\r\n<hr><center>nginx/1.20.0</center>\r\n</body>\r\n</html>\r\n","http_code":403,"headers":{"Server":"nginx/1.20.0","Date":"Thu, 06 May 2021 09:00:14 GMT","Content-Length":"153","Content-Type":"text/html","Connection":"keep-alive"}},"producer":{"modsecurity":"ModSecurity v3.0.4 (Linux)","connector":"ModSecurity-nginx v1.0.1","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Method is not allowed by policy","details":{"match":"Matched \"Operator `Within' with parameter `GET HEAD POST OPTIONS' against variable `REQUEST_METHOD' (Value: `DELETE' )","reference":"v0,6","ruleId":"911100","file":"/etc/nginx/modsec/crs/owaspcrs/rules/REQUEST-911-METHOD-ENFORCEMENT.conf","lineNumber":"27","data":"DELETE","severity":"2","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-generic","OWASP_CRS/POLICY/METHOD_NOT_ALLOWED","WASCTC/WASC-15","OWASP_TOP_10/A6","OWASP_AppSensor/RE1","PCI/12.1"],"maturity":"9","accuracy":"9"}},{"message":"Inbound Anomaly Score Exceeded (Total Score: 1000)","details":{"match":"Matched \"Operator `Ge' with parameter `5' against variable `TX:ANOMALY_SCORE' (Value: `1000' )","reference":"","ruleId":"949110","file":"/etc/nginx/modsec/crs/owaspcrs/rules/REQUEST-949-BLOCKING-EVALUATION.conf","lineNumber":"44","data":"","severity":"2","ver":"","rev":"","tags":["application-multi","language-multi","platform-multi","attack-generic"],"maturity":"0","accuracy":"0"}}]}}",
    "@timestamp": "2021-05-06T11:01:29.155Z"
  },
  "fields": {
    "@timestamp": [
      "2021-05-06T11:01:29.155Z"
    ]
  }
}
Tags:

2

Answers


  1. Chosen as BEST ANSWER
    0

    Ok - I resolved it with below changes in config file.

    filter {
        split {
        field => "[transaction][messages]"
        }
      }

  2. 0

    The type setting in the file input doesn’t work this way. It’s a legacy setting that basically shouldn’t be used, and refers to the elasticsearch document type (which is now a deprecated concept) rather than the type of input file.

    You might get better results using the JSON codec

    input {
      file {
        codec => "json"
        path => "/var/log/modsec_audit.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
      }
    }

    In this case, you’ll almost certainly want to remove the json filter.

    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search