I am trying to build API using flask which will use my saved ML model. The model was built using sklearn, pipeline and an helper function(lemmatizer_preprocessing), and stored in pickle format using joblib Now, when I try to use that model to build my flask app it given the Attribute error
AttributeError: module '__main__' has no attribute 'lemmatizer_preprocessing'
Code which used to build the model and save it
def lemmatizer_preprocessing(mess):
nopunc = [char for char in mess if char not in string.punctuation]
nopunc = ''.join(nopunc)
nopunc = [lemmatizer.lemmatize(word) for word in nopunc.split()]
nopunc = [word for word in nopunc if word.lower() not in stopwords.words('english')]
temp = ' '.join(nopunc).strip()
return re.sub(r'[^\w]', ' ', temp)
....
....
....
pipeline1 = Pipeline([
('bow', CountVectorizer(analyzer=lemmatizer_preprocessing)),
('classifier', MultinomialNB()),
...
])
....
....
....
joblib.dump(pipeline1, 'filename.pkl')
Now whenever I try to import this model, it shows the above error. I know it is showing error as the function lemmatizer_preprocessing is required by joblib to properly deserialize the model, but for some reason the function is not being registered.
I am using two files to divide the code for my flask app , app.py and predictor.py
Code for app.py:
from flask import Flask, jsonify, request, make_response
from predictor import predict_jihad
app = Flask(__name__, instance_relative_config=True)
predict_jihad = predict_jihad()
@app.errorhandler(404)
def not_found(error):
return make_response(jsonify({'error': 'Not found'}), 404)
@app.errorhandler(500)
def not_found(error):
return make_response(jsonify({'error': 'Not found'}), 500)
@app.route('/')
def index():
text = request.args.get('text')
if type(text) is str and len(text)!=0:
return jsonify({"probability":predict_jihad.get_prediction(text)})
else:
return jsonify({"error":"check passed value"})
app.run(debug=False)
Code for predictor.py:
from nltk.corpus import stopwords
import string
from sklearn.feature_extraction.text import TfidfTransformer ,CountVectorizer
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import MultinomialNB
import re
from nltk.stem import WordNetLemmatizer
import joblib
class predict_jihad:
def __init__(self):
super().__init__()
lemmatizer = WordNetLemmatizer()
file = './filename.pkl'
def deserialize(self):
def lemmatizer_preprocessing(mess):
lemmatizer = WordNetLemmatizer()
nopunc = [char for char in mess if char not in string.punctuation]
nopunc = ''.join(nopunc)
nopunc = [self.lemmatizer.lemmatize(word) for word in nopunc.split()]
nopunc = [word for word in nopunc if word.lower() not in stopwords.words('english')]
temp = ' '.join(nopunc).strip()
return re.sub(r'[^\w]', ' ', temp)
model = joblib.load(open('filename.pkl','rb'))
return model
def get_prediction(self,text):
model = self.deserialize()
return model.predict_proba([text])[0][1]
All other files are in-place and no other error is being registered. Please provide solutions.