How do you use Lucene.Net phonetic search

Viewed 370

The Lucene.Net documentation is extremely lacking - I've been trying to figure out how to make use of the Lucene.Net phonetic searching functionality.

I've mainly been referencing the lucene.ne git project's test case for the PhoneticFilter: https://github.com/apache/lucenenet/blob/master/src/Lucene.Net.Tests.Analysis.Phonetic/TestPhoneticFilter.cs

How I create my index:

// add new index entry
Document doc = new Document
{
    new TextField("brand", vehicle.Brand ?? string.Empty, Field.Store.YES),
    new TextField("range", vehicle.Range ?? string.Empty, Field.Store.YES),
    new TextField("model", vehicle.Model ?? string.Empty, Field.Store.YES),
    new TextField("year", vehicle.Year ?? string.Empty, Field.Store.YES),
};

// add entry to index
writer.AddDocument(doc);

Next, from the test case, it seems you can add the PhoneticFilter to the Analyzer:

// set up lucene searcher
using DirectoryReader reader = DirectoryReader.Open(Directory);

Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
    Tokenizer tokenizer = new KeywordTokenizer(reader);
    return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, new Soundex(), false));
});

IndexSearcher searcher = new IndexSearcher(reader);
int hits_limit = searchModel.Start + searchModel.Qty;
TopDocs topDocs;
Sort sortOrder = null;

BooleanQuery vehicleFilterQuery = new BooleanQuery();

var brandParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "brand", analyzer);
var brandQuery = ParseQuery(searchModel.SearchTerm, brandParser);
brandQuery.Boost = 4.0f;
vehicleFilterQuery.Add(brandQuery, Occur.SHOULD);

var rangeParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "range", analyzer);
var rangeQuery = ParseQuery(searchModel.SearchTerm, rangeParser);
rangeQuery.Boost = 3.0f;
vehicleFilterQuery.Add(rangeQuery, Occur.SHOULD);

var modelParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "model", analyzer);
var modelQuery = ParseQuery(searchModel.SearchTerm, modelParser);
modelQuery.Boost = 2.0f;
vehicleFilterQuery.Add(modelQuery, Occur.SHOULD);

var yearParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "year", analyzer);
var yearQuery = ParseQuery(searchModel.SearchTerm, yearParser);
yearQuery.Boost = 1.0f;
vehicleFilterQuery.Add(yearQuery, Occur.SHOULD);

topDocs = searcher.Search(vehicleFilterQuery, null, hits_limit, sortOrder ?? Sort.RELEVANCE);
ScoreDoc[] subset = topDocs.ScoreDocs.Skip(searchModel.Start).Take(searchModel.Qty).ToArray();

return MapToModelList(subset, searcher);

After implementing this in order to test, I search with Audee, expecting to get a bunch of Audi results, but while the regular (correctly spelled) search works, the phonetic search seems to have no impact.

I've tried the Metaphone(), DoubleMetaphone(), Soundex(), RefinedSoundex(), Caverphone1() and Caverphone2() encoders

Metaphone https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.Metaphone.html

DoubleMetaphone https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.DoubleMetaphone.html

Soundex https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.Soundex.html

RefinedSoundex https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.RefinedSoundex.html

Caverphone1 https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.Caverphone1.html

Caverphone2 https://lucenenetdocs.azurewebsites.net/api/Lucene.Net.Analysis/Lucene.Net.Analysis.Phonetic.Language.Caverphone2.html

1 Answers

There is a specialized DoubleMetaphoneFilter designed for basic phonetic matches. You only need to use the PhoneticFilter if you need explicit control over the phonetic algorithm being used.

Here is an example (based on this StackOverflow answer):

// Ensures index backward compatibility
const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48;

[Test]
public void DoubleMetaphoneExample()
{
    // Create a directory
    var directory = new RAMDirectory();

    // Create a phonetic analyzer
    var analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => {
        var tokenizer = new KeywordTokenizer(input: reader);
        var stream = new DoubleMetaphoneFilter(input: tokenizer, maxCodeLength: 8, inject: false);
        return new TokenStreamComponents(tokenizer, stream);
    });

    // Create an index writer
    var indexConfig = new IndexWriterConfig(AppLuceneVersion, analyzer);
    using (var writer = new IndexWriter(directory, indexConfig))
    {
        // Add documents
        foreach (var vehicle in Vehicles)
        {
            // add new index entry
            Document doc = new Document
            {
                new TextField("brand", vehicle.Brand ?? string.Empty, Field.Store.YES),
                new TextField("range", vehicle.Range ?? string.Empty, Field.Store.YES),
                new TextField("model", vehicle.Model ?? string.Empty, Field.Store.YES),
                new TextField("year", vehicle.Year ?? string.Empty, Field.Store.YES),
            };

            // add entry to index
            writer.AddDocument(doc);
        }
    }
    // Done indexing

    // Begin Search

    var searchModel = new { SearchTerm = "audee" };

    // Open an IndexReader
    using var reader = DirectoryReader.Open(directory);

    IndexSearcher searcher = new IndexSearcher(reader);
    int hits_limit = 5;
    TopDocs topDocs;
    Sort sortOrder = null;

    BooleanQuery vehicleFilterQuery = new BooleanQuery();

    var brandParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "brand", analyzer);
    var brandQuery = ParseQuery(searchModel.SearchTerm, brandParser);
    brandQuery.Boost = 4.0f;
    vehicleFilterQuery.Add(brandQuery, Occur.SHOULD);

    var rangeParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "range", analyzer);
    var rangeQuery = ParseQuery(searchModel.SearchTerm, rangeParser);
    rangeQuery.Boost = 3.0f;
    vehicleFilterQuery.Add(rangeQuery, Occur.SHOULD);

    var modelParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "model", analyzer);
    var modelQuery = ParseQuery(searchModel.SearchTerm, modelParser);
    modelQuery.Boost = 2.0f;
    vehicleFilterQuery.Add(modelQuery, Occur.SHOULD);

    var yearParser = new QueryParser(Lucene.Net.Util.LuceneVersion.LUCENE_48, "year", analyzer);
    var yearQuery = ParseQuery(searchModel.SearchTerm, yearParser);
    yearQuery.Boost = 1.0f;
    vehicleFilterQuery.Add(yearQuery, Occur.SHOULD);

    topDocs = searcher.Search(vehicleFilterQuery, null, hits_limit, sortOrder ?? Sort.RELEVANCE);

    // topDocs.ScoreDocs contains

    // doc = 2
    // doc = 3
    // doc = 4
    // doc = 7
}

private static Search.Query ParseQuery(string searchTerm, QueryParser queryParser)
{
    return queryParser.Parse(searchTerm);
}

private static Vehicle[] Vehicles = new Vehicle[] {
    new Vehicle { Brand = "Ford", Model = "Taurus", Year = "1986" },
    new Vehicle { Brand = "Ford", Model = "Fiesta", Year = "1990" },
    new Vehicle { Brand = "Audi", Model = "A4 45", Year = "2021" },
    new Vehicle { Brand = "Audi", Model = "Q3 45 S", Year = "2021" },
    new Vehicle { Brand = "Audie", Model = "Q3 45 S", Year = "2021" },
    new Vehicle { Brand = "Toyota", Model = "Corolla", Year = "2010" },
    new Vehicle { Brand = "Toyota", Model = "Hilux", Year = "2015" },
    new Vehicle { Brand = "Audi", Model = "A4", Year = "2017" },
};

public class Vehicle
{
    public string Brand { get; set; }
    public string Range { get; set; }
    public string Model { get; set; }
    public string Year { get; set; }
}

Alternatively, you can use PhoneticFilter to pick an algorithm. I tried your example with DoubleMetaphone (both with and without setting MaxCodeLen) and it worked fine.

var stream = new PhoneticFilter(
    input: tokenizer,
    encoder: new DoubleMetaphone() { MaxCodeLen = 8 },
    inject: false);

There is some general information about phonetic algorithms on Wikipedia. In most cases, Double Metaphone is the best choice for the English language.

Related