T
T
Tylen2021-04-08 16:30:47
elasticsearch
Tylen, 2021-04-08 16:30:47

How to filter traffic in logstash?

There are these logs:

{
  "_index": "filebeat-6.8.15-2021.04.08",
  "_type": "_doc",
  "_id": "9AKesXgB3KIPxKt1f5zA",
  "_score": 1,
  "_source": {
    "requset_id": "[3b358c6e5fa35f13a71c5ab194754199]",
    "timestamp": "2021-04-08T16:12:34.389334",
    "@timestamp": "2021-04-08T13:12:35.082Z",
    "tags": [
      "sed2.rtech.ru",
      "beats_input_codec_plain_applied"
    ],
    "input": {
      "type": "log"
    },
    "beat": {
      "name": "sed2.rtech.ru",
      "hostname": "sed2.rtech.ru",
      "version": "6.8.15"
    },
    "@version": "1",
    "message": "INFO, [2021-04-08T16:12:34.389334 #22938]  --  [3b358c6e5fa35f13a71c5ab194754199] [ES Query] GET http://cluster.elastic.sed2.rtech.ru:9200/incoming_documents_production_db_core_cards_incoming_documents_cards_2021,incoming_documents_production_db_core_cards_incoming_documents_cards_no_year/_search?from=0&size=10 [status:200, request:0.883s, query:0.003s]",
    "host": {
      "containerized": false,
      "name": "sed2.rtech.ru",
      "architecture": "x86_64",
      "os": {
        "name": "Ubuntu",
        "family": "debian",
        "codename": "trusty",
        "platform": "ubuntu",
        "version": "14.04.6 LTS, Trusty Tahr"
      },
      "id": "fbe96c6e791e31d5a2f7e323580db99c"
    },
    "pid": "#22938",
    "prospector": {
      "type": "log"
    },
    "source": "/home/esd-admin/apps/cards/current/incoming_documents/log/production.log",
    "log": {
      "file": {
        "path": "/home/esd-admin/apps/cards/current/incoming_documents/log/production.log"
      }
    },
    "msg": " [ES Query] GET http://cluster.elastic.sed2.rtech.ru:9200/incoming_documents_production_db_core_cards_incoming_documents_cards_2021,incoming_documents_production_db_core_cards_incoming_documents_cards_no_year/_search?from=0&size=10 [status:200, request:0.883s, query:0.003s]",
    "level": "INFO",
    "offset": 920843638
  },
  "fields": {
    "@timestamp": [
      "2021-04-08T13:12:35.082Z"
    ],
    "timestamp": [
      "2021-04-08T16:12:34.389Z"
    ]
  }
}

Parse the logs, with the condition where the file comes from, if the path from where the log came from matches the path that I entered, then the log was parsed, here is the pattern:
filter {
   if "/home/esd-admin/apps/background_jobs/current/log/production.log" in [source] or "/home/esd-admin/apps/cards/current/incoming_orders/log/production.log" in [source] {
   grok {
     match => { "message" => "(?<level>^\w+)\W+(?<timestamp>\s\d+-\d+-\w+:\w+:\w+.\w+|\[(\d+\d+-\d+-\w+:\w+:\w+.\w+)).(?<pid>#\d+).+(?<requset_id>\[\w+]).+(?<tag>\[\w+\s\w+\]).(?<method>[A-Z]+\s)(?<url>http\S+)\s\[status:(?<status>\d+),\srequest:(?<request>(\d+\.\w+)),\squery:(?<query>.+)]"}
    }
  } else{
   grok {
     match => { "message" => "(?<level>^\w+)\W+(?<timestamp>\d+-\d+-\w+:\w+:\w+.\w+|\[(\d+\d+-\d+-\w+:\w+:\w+.\w+)).(?<pid>#\d+)\D+(?<requset_id>\[\w+])(?<msg>.+)"}
    }
}
}

Now the logs are not parsed

Answer the question

In order to leave comments, you need to log in

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question