I have a file containing 14 million records each with 2 fields. I had indexed the documents with below config:
private fun getIndexWriter(directory: String): IndexWriter = try {
val indexWriterConfig = IndexWriterConfig(KeywordAnalyzer())
indexWriterConfig.ramBufferSizeMB = 2048.0
indexWriterConfig.openMode = IndexWriterConfig.OpenMode.CREATE_OR_APPEND
IndexWriter(FSDirectory.open(Paths.get(directory)), indexWriterConfig)
} catch (exception: Exception) {
logger.error("File missing: {} ", exception.message)
throw Exception("File missing: ${exception.message}")
}
fun indexDocs(documents: List<Document>)= mutex.withLock {
coroutineScope {
val indexedDocs = getIndexWriter("directory_path").use { indexWriter ->
try {
documents.forEach { document ->
try {
indexWriter.addDocument(document)
} catch (exception: IOException) {
logger.error("Failed to add document: {} to passive index", document)
}
}
indexWriter.flush()
indexWriter.commit()
} catch (exception: IOException) {
logger.error("Failed to commit")
}
indexWriter.docStats.numDocs
}
}
}
It takes 5 minutes around to complete the index and to start the application, however I wanted to check if there can be some optimization done to reduce the time taken for indexing.
Thanks for the help in advance!