I am using elasticsearch for log analysis, and some fields have large length: PROCESS_INFO.LOG_INFO.MSG and PROCESS_INFO.SERVICE_INFO.EXCEPTION.
I mapped those fields with 'keyword' type with ignore_above mapping (ignore_above: 7864320)
Even after I have set dynamic mapping and new documents were inserted in a new index, exception 'UTF8 encoding is longer than the max length 32766' still exists.
Any advice, please? Thank you in advance!
- I did not map field PROCESS_INFO.LOG_INFO.MSG with 'text' field, because worried about too much indexing.
- Error trace:
ElasticsearchStatusException[Elasticsearch exception [type=illegal_argument_exception, reason=Document contains at least one immense term in field="PROCESS_INFO.LOG_INFO.MSG" (whose UTF8 encoding is longer than the max length 32766), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '[91, 50, 48, 50, 48, 45, 48, 55, 45, 49, 54, 95, 50, 51, 58, 53, 54, 58, 53, 56, 46, 56, 49, 51, 50, 51, 53, 93, 32, 76]...', original message: bytes can be at most 32766 in length; got 191323]]; nested: ElasticsearchException[Elasticsearch exception [type=max_bytes_length_exceeded_exception, reason=bytes can be at most 32766 in length; got 191323]];
at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:177)
at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:1721)
at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:1698)
at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1461)
at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1418)
at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1388)
at org.elasticsearch.client.RestHighLevelClient.index(RestHighLevelClient.java:836)
at com.lgcns.mom.process.LogServer.insertSingleDocument(LogServer.java:196)
at com.lgcns.mom.process.LogServer.executeService(LogServer.java:176)
at com.lgcns.mom.process.QueueServer$1.onReceive(QueueServer.java:93)
at com.solacesystems.jcsmp.impl.flow.FlowHandleImpl.triggerAsyncDelivery(FlowHandleImpl.java:1534)
GET /_mapping
{
"servicerunlogreport-myprogram-2020.07.17" : {
"mappings" : {
"dynamic" : "true",
"_meta" : { },
"_source" : {
"includes" : [ ],
"excludes" : [ ]
},
"dynamic_date_formats" : [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z||yyyy/MM/dd HH:mm:ss.SSS"
],
"dynamic_templates" : [
{
"strings_as_keywords" : {
"match_mapping_type" : "string",
"mapping" : {
"type" : "keyword"
}
}
}
],
"date_detection" : true,
"numeric_detection" : false,
"properties" : {
"CMD" : {
"type" : "keyword"
},
"PROCESS_INFO" : {
"properties" : {
"LOG_INFO" : {
"properties" : {
"ERR_MSG" : {
"type" : "keyword"
},
"MSG" : {
"type" : "keyword",
"ignore_above" : 7864320
},
"SERVER_TYPE" : {
"type" : "keyword"
},
"SEV" : {
"type" : "keyword"
},
"TXN_KEY" : {
"type" : "keyword"
},
"TXN_NAME" : {
"type" : "keyword"
},
"TXN_TIME" : {
"type" : "date",
"format" : "strict_date_optional_time"
}
}
},
"PROCESS_ID" : {
"type" : "keyword"
},
"PROCESS_TYPE" : {
"type" : "keyword"
},
"PROCESS_TYPE_DETAIL" : {
"type" : "keyword"
},
"SERVER_ID" : {
"type" : "keyword"
},
"SERVICE_INFO" : {
"properties" : {
"EXCEPTION" : {
"type" : "keyword",
"ignore_above" : 7864320
}
}
},
"SYSTEM_ID" : {
"type" : "keyword"
}
}
}
}
}
}
}