How to create a confusion matrix for a NER Spacy model?

Viewed 76

I want to develop a confusion matrix for my model, but I'm not sure how to go about it or which variable to use. Since my model has two functions one for training and the other for testing I'm not sure if I should make the confusion matrix for both sets of results or just for testing.

Here is the function to do the training

def training(training_data, nlp, batch_size, iteration_index):
    # batch up the examples using spaCy's minibatch
    losses = {}
    batches = minibatch(training_data, size=batch_size)
    for batch in batches:
        for text, annotations in batch:
            doc = nlp.make_doc(text)
            example = Example.from_dict(doc, annotations)
            # Update the model
            nlp.update([example], losses=losses, drop=0.3)
    mlflow.log_metrics(losses, step=iteration_index)
    return nlp

This function allows me to test the model

def testing(testing_data, nlp, iteration_index):
    testing_examples = []
    scorer_example = []
    for text, annotations in testing_data:
        doc = nlp.make_doc(text)
        testing_examples.append(Example.from_dict(doc, annotations))

    scorer_example = nlp.evaluate(testing_examples)
    del scorer_example["ents_per_type"]
    mlflow.log_metrics(scorer_example, step=iteration_index)

And to complete the loop

with nlp.disable_pipes(*unaffected_pipes):
    with mlflow.start_run(experiment_id=experiment_id, run_name=model_name):
        mlflow.set_tag("model_flavor", model_name)
        mlflow.log_param("BATCH_SIZE", BATCH_SIZE)
        mlflow.log_param("TRAINING_ITERATION", TRAINING_ITERATION)
        mlflow.log_param("LABELS", CLASSES)
        mlflow.log_param("PIPE_NAMES", nlp.pipe_names)

        # Training for many iterations
        for iteration in tqdm(range(TRAINING_ITERATION)):
            
            # shuufling examples before every iteration
            random.shuffle(train_data)

            nlp = training(train_data, nlp, BATCH_SIZE, iteration)
            testing(test_data, nlp, iteration)

        #Save results in MLflow
        mlflow.log_artifact(local_path = './ner.ipynb')
        mlflow.spacy.log_model(spacy_model=nlp, artifact_path=str(train_name))

Preferably I would like to use Plotly because I already had the opportunity to use it in another project

0 Answers
Related