We have a cluster running with a single index with embeddings indexed. We are performing knn search on this cluster.
Cluster document size = 54M records
Shards = 10
Embeddings size = 64 dimension
I have tried both a-knn (approximate knn) and script_score knn_score by applying pre-filter.
In case of a-knn, it takes too much time and doesn't respond at all (on kibana). On aws dashboard, latencies are > 60 seconds (yes, seconds)
In case script score knn, I have applied a filter to give documents within 2km radius (around 50k documents), then it gives results within 400ms. If I just increase the increase the pre-filter radius to 2.5km (around 100k) records, latencies shoot up again.
Here are the settings/mappings and kibana query
{
"search_item_embedding_v2": {
"aliases": {
"search_item_embedding": {}
},
"mappings": {
"properties": {
"cityId": {
"type": "integer"
},
"embeddingsV1": {
"type": "knn_vector",
"dimension": 64
},
"embeddingsV2": {
"type": "knn_vector",
"dimension": 64
},
"enabled": {
"type": "boolean"
},
"id": {
"type": "text"
},
"isLongDistanceEnabled": {
"type": "boolean"
},
"isRestEnabled": {
"type": "boolean"
},
"latLong": {
"type": "geo_point"
},
"name": {
"type": "text"
},
"restId": {
"type": "integer"
},
"restName": {
"type": "text"
},
"vegIdentifier": {
"type": "integer"
},
"version": {
"type": "long"
}
}
},
"settings": {
"index": {
"search": {
"slowlog": {
"level": "info",
"threshold": {
"fetch": {
"warn": "90ms",
"debug": "30ms",
"info": "40ms"
},
"query": {
"warn": "90ms",
"debug": "30ms",
"info": "40ms"
}
}
}
},
"refresh_interval": "60s",
"indexing": {
"slowlog": {
"level": "info",
"threshold": {
"index": {
"warn": "500ms",
"debug": "100ms",
"info": "300ms"
}
},
"source": "1000"
}
},
"number_of_shards": "10",
"auto_expand_replicas": "0-all",
"blocks": {
"read_only_allow_delete": "false",
"write": "false"
},
"provided_name": "search_item_embedding_v2",
"knn": "true",
"creation_date": "1638438116281",
"number_of_replicas": "1",
"uuid": "-tzXpNRfTY-hTBvu_NKPIA",
"version": {
"created": "135217827"
}
}
}
}
}
Kibana query (method 2):
{
"size": 100,
"query": {
"script_score": {
"query": {
"bool": {
"filter": [
{
"term": {
"isRestEnabled": {
"value": true,
"boost": 1
}
}
},
{
"term": {
"enabled": {
"value": true,
"boost": 1
}
}
},
{
"geo_distance": {
"latLong": [
77.693052,
12.928925
],
"distance": "2km",
"distance_type": "arc",
"validation_method": "STRICT",
"ignore_unmapped": false,
"boost": 1
}
}
]
}
},
"script": {
"source": "knn_score",
"lang": "knn",
"params": {
"field": "embeddingsV1",
"query_value": [],
"space_type": "cosinesimil"
}
}
}
},
"_source": {
"includes": [
"id",
"restId"
]
}
}