I have below type of events. I'm trying to split field's key, value as new event. I'm able to do it for two fields(TOTAl_VOLUME, SUCCESS_VOLUME), but when I try for 3rd field, logstash stop responding.(Split doesn't work for more than 2 fields).
{
"agentId" => "Log_Agent",
"@metadata" => {
"A1EvtFingerprint" => "AGENTID=Log_Agent&TIME=1657708200000&RESPTYPE=DC",
"indexname" => "heal_collated_agent_txn",
"tablename" => "agent_transactions_data",
"accountid" => "mle_account",
"enable_rubydebug" => "true"
},
"total" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588
},
"success" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588
},
"max_response" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.011000156402588
},
"response_type" => "DC",
"aggLevelInMins" => 15,
"timeInGMT" => 1657708200000,
"avg_response" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.5954742431640625,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.6110687255859375,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 4.580192565917969
},
"timeout" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 777,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 839,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 781
},
"unknown" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 773,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 794,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 746
},
"fail" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 770,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 737,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 800
},
"@timestamp" => 2022-07-13T10:30:00.000Z,
"slow" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 782,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 788,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 744
},
"min_response" => {
"GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0,
"GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0,
"GET#/txn/branchserver23.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12" => 5.0
},
"@version" => "1"
}
Desired output should be(Splitted multiple field value into multiple events)
{
"txnId" : "GET#/txn/branchserver50.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12",
"timeInGMT" : 1657048320000,
"dcKpis" : {
"UNKNOWN_VOLUME" : 773,
"TIMEOUT_VOLUME" : 777,
"FAIL_VOLUME" : 770,
"MIN_RESPONSE_TIME" : 50,
"TOTAL_VOLUME" : 5.011000156402588,
"AVG_RESPONSE_TIME" : 4.5954742431640625,
"MAX_RESPONSE_TIME" : 5.011000156402588,
"SUCCESS_VOLUME" : 5.011000156402588,
"SLOW_VOLUME" : 782
}
},
{
"txnId" : "GET#/txn/branchserver51.aspx|srv=73689505-0ca6-48fe-a4da-4cf7ed4acd82|acc=12",
"timeInGMT" : 1657048320000,
"dcKpis" : {
"UNKNOWN_VOLUME" : 794,
"TIMEOUT_VOLUME" : 839,
"FAIL_VOLUME" : 737,
"MIN_RESPONSE_TIME" : 5.0,
"TOTAL_VOLUME" : 5.011000156402588,
"AVG_RESPONSE_TIME" : 4.6110687255859375,
"MAX_RESPONSE_TIME" : 5.011000156402588,
"SUCCESS_VOLUME" : 5.011000156402588,
"SLOW_VOLUME" : 788
}
}
Following is my pipeline:
ruby {
code => '
values = event.get("total")
if values.is_a? Hash
someField1 = []
values.each { |k, v|
someField1 << { "txnId1" => k, "total" => v }
}
event.set("someField1", someField1)
end
event.remove("total")
'
}
ruby {
code => '
values = event.get("success")
if values.is_a? Hash
someField2 = []
values.each { |k, v|
someField2 << { "txnId2" => k, "success" => v }
}
event.set("someField2", someField2)
end
event.remove("success")
'
}
split {
field => 'someField1'
}
split {
field => 'someField2'
}
mutate {
rename => {
"[someField1][txnId1]" => "[@metadata][txnId1]"
"[someField1][total]" => "[dcKpis][TOTAl_VOLUME]"
"[someField2][txnId2]" => "[@metadata][txnId2]"
"[someField2][success]" => "[dcKpis][SUCCESS_VOLUME]"
}
remove_field => ["someField1","someField2","someField3","someField4","someField5","someField6","someField7","someField8","someField9"]
}