spaCy - Add extended function to pipeline causes stack overflow

user_192204 0
Viewed 80

I am trying to add a matcher rule-based function to my spaCy pipeline. However, adding it to the pipeline is causing a StackOverflow error. High probability it is a user error. Any suggestions or thoughts would be greatly appreciated.

Running the function without adding it to the pipeline works fine.

Code Example:

import spacy
from spacy.matcher import PhraseMatcher
from spacy.tokens import Span

nlp = spacy.load("en_core_web_sm")

def extend_matcher_entities(doc):
    matcher = PhraseMatcher(nlp.vocab, attr="SHAPE")
    matcher.add("TIME", None, nlp("0305Z"), nlp("1315z"),nlp("0830Z"),nlp("0422z"))

    new_ents = []
    for match_id, start, end in matcher(doc):
        new_ent = Span(doc, start, end, label=nlp.vocab.strings[match_id])
        new_ents.append(new_ent)
    
    doc.ents = new_ents
    return doc

# Add the component after the named entity recognizer
nlp.add_pipe(extend_matcher_entities, after='ner')

doc = nlp("At 0560z, I walked over to my car and got in to go to the grocery store.")

# extend_matcher_entities(doc)
print([(ent.text, ent.label_) for ent in doc.ents])

This example from spacy code examples works fine:

import spacy
from spacy.tokens import Span

nlp = spacy.load("en_core_web_sm")

def expand_person_entities(doc):
    new_ents = []
    for ent in doc.ents:
        if ent.label_ == "PERSON" and ent.start != 0:
            prev_token = doc[ent.start - 1]
            if prev_token.text in ("Dr", "Dr.", "Mr", "Mr.", "Ms", "Ms."):
                new_ent = Span(doc, ent.start - 1, ent.end, label=ent.label)
                print(new_ent)
                new_ents.append(new_ent)
        else:
            new_ents.append(ent)
    doc.ents = new_ents
    print(new_ents)
    return doc

# Add the component after the named entity recognizer
nlp.add_pipe(expand_person_entities, after='ner')

doc = nlp("Dr. Alex Smith chaired first board meeting of Acme Corp Inc.")
print([(ent.text, ent.label_) for ent in doc.ents])

What am I missing?

spacy python
1 Answers

The offending line due to which you have a circular reference is this one:

matcher.add("TIME", None, nlp("0305Z"), nlp("1315z"),nlp("0830Z"),nlp("0422z"))

Take it out of your function defintion and you're fine to go:

import spacy
from spacy.matcher import PhraseMatcher
from spacy.tokens import Span

nlp = spacy.load("en_core_web_sm")
pattern = [nlp(t) for t in ("0305Z","1315z","0830Z","0422z")]


def extend_matcher_entities(doc):
    matcher = PhraseMatcher(nlp.vocab, attr="SHAPE")
    matcher.add("TIME", None, *pattern)

    new_ents = []
    for match_id, start, end in matcher(doc):
        new_ent = Span(doc, start, end, label=nlp.vocab.strings[match_id])
        new_ents.append(new_ent)
    
    doc.ents = new_ents
    # doc.ents = list(doc.ents) + new_ents
    return doc

# Add the component after the named entity recognizer
nlp.add_pipe(extend_matcher_entities, after='ner')

doc = nlp("At 0560z, I walked over to my car and got in to go to the grocery store.")

# extend_matcher_entities(doc)
print([(ent.text, ent.label_) for ent in doc.ents])
[('0560z', 'TIME')]

Note as well, by doc.ents = new_ents you're overwriting any entities extracted before

user_43170580
Related