I am trying to fetch the event.get(sessions) value from my config file, but I am not getting complete thing. I am having one enum-pieline.conf which is having input from elastic search index.
input {
elasticsearch {
hosts => "localhost:9200"
index => "enum_pipeline_1_sessionid_index_2"
size => 250
scroll => "1m"
query => '{ "sort": ["_doc"] }'
docinfo => true
}
}
filter {
elasticsearch {
hosts => ["http://localhost:9200"]
index => "message_index"
query_template => "./getSessionMsgsQuery.json"
fields => {
"message" => "sessions"
}
}
elasticsearch {
hosts => ["http://localhost:9200"]
index => "enum-sessionid-counter"
query_template => "./lookup.json"
fields => {
"additionalData" => "additionalData"
"lastSession" => "lastSession"
}
}
ruby{
path => "./enum-ruby-filter-2.rb"
}
}
output{
if [Exception] {
elasticsearch {
hosts => ["localhost:9200"]
index => "pipeline-two-out"
}
} else if [additionalData] {
elasticsearch {
hosts => ["localhost:9200"]
index => "pipeline-two-out"
document_id => "%{[additionalData][doc_id]}"
scripted_upsert => true
action => "update"
script => '
ctx._source.additionalData = params.event.get("additionalData");
ctx._source.lastSession = params.event.get("lastSession");
'
}
} else {
elasticsearch {
hosts => ["localhost:9200"]
index => "pipeline-two-out"
document_id => "%{[@metadata][updated_doc_id]}"
}
}
}
RUBY FILTER
def filter(event)
begin
print(event)
index_name = 'message_index'
event_doc = event.get('sessions')
print "\n---****---\n"
print event_doc
print "\n---****---\n"
#When the elasticsearch lookup returns single document
#the returned type will be doc and if it contains multiple
#docs the return type will be array
if event_doc.is_a?(Hash)
event_doc = [event_doc]
elsif event_doc.is_a?(Array)
event_doc = event_doc.sort_by{ |hash| hash['timestamp'] }
end
#event_doc = event_doc.sort_by{ |hash| hash['timestamp'] }
doc_id = event.get('doc_id')
print "\n---**---\n"
print (doc_id)
print "\n---**---\n"
additionalData = event.get('additionalData')
print "\n---**---\n"
print (additionalData)
print "\n---**---\n"
if additionalData == nil
sessionDoc = {}
sessionIdCounter = 1
recvdResponse = true
else
sessionIdCounter = additionalData['sessionIdCounter']
recvdResponse= additionalData['recvdResponse']
sessionDoc = event.get('lastSession')
end
event_array = []
event_doc.each { |message|
message_type = message['MessageType']
if message_type == 'ENUMResponse'
sessionDoc[message_type] = message
updated_doc_id = doc_id + '-' + sessionIdCounter.to_s
sessionDoc['@metadata'] = {'index_name' => index_name,
'updated_doc_id' => updated_doc_id,
'doc_id' => doc_id
}
sessionDoc['sessionIdCounter'] = sessionIdCounter
event_array.append(LogStash::Event.new(sessionDoc))
sessionIdCounter += 1
recvdResponse = true
sessionDoc = {}
elsif message_type == 'ENUMRequest' and recvdResponse == false
updated_doc_id = doc_id + '-' + sessionIdCounter.to_s
sessionDoc['@metadata'] = {'index_name' => index_name,
'updated_doc_id' => updated_doc_id, 'sessionIdCounter' => sessionIdCounter,
'doc_id' => doc_id
}
sessionDoc['sessionIdCounter'] = sessionIdCounter
event_array.append(LogStash::Event.new(sessionDoc))
event_array.append(LogStash::Event.new(sessionDoc))
sessionIdCounter += 1
sessionDoc = {}
sessionDoc[message_type] = message
else
recvdResponse = false
sessionDoc[message_type] = message
end
}
additionalData = {
'sessionIdCounter' => sessionIdCounter,
'doc_id' => doc_id,
'recvdResponse' => recvdResponse
}
last_doc = { 'additionalData' => additionalData, 'lastSession' => sessionDoc }
event_array.append(LogStash::Event.new(last_doc))
return event_array
rescue Exception => e
event_array = []
event_doc = {"Exception" => e.message}
event_doc["backtrace"] = e.backtrace.inspect
event_array.append(LogStash::Event.new(event_doc))
return event_array
end
end
In my elastic search I have 4 index are present
1. message_index--> Having response/request in it
2.enum_pipeline_1_sessionid_index_2 --> havind document id(session id)
3.counter_index & pipeline-two-out--> used for output purposes
When I am trying to debug I am not getting my session in the event.get(sessions).
OUTPUT
2022-09-14T15:07:32.956Z %{host} %{message}
---****---
---****---
------ 0x486b ------
---**---
---**--- 2022-09-14T15:07:32.972Z %{host} %{message} ---****---
---****---
------ 0x486b ------
---**---
---**---
The fields of event_doc and additionalData is empty. Please help almost spent 2 days in it already.
What I have done
Changed my ruby filter code and conf file too but no progress.