How to save ngrams generator results in a text file?

user_3729357 0
Viewed 510

I am extracting ngrams from a corpus using nltk and python and I need to save the generated ngrams in a text file.

I tried this code but no result:

import nltk, re, string, collections
from nltk.util import ngrams 
with open("titles.txt", "r", encoding='utf-8') as file:
    text = file.read()
tokenized = text.split()
Monograms = ngrams(tokenized, 1)
MonogramFreq = collections.Counter(Monograms)
with open('output.txt', 'w') as f:    
   f.write(str(MonogramFreq))

here is a sample of titles.txt:

Joli appartement s3 aux jardins de carthage mz823
Villa 600m2 haut standing à hammamet
Hammem lif
S2 manzah 7
Terrain constructible de 252m2 clôturé
Terrain nu a gammarth
Terrain agrecole al fahes
Bureau 17 pièces
Usine 5000m2 mannouba

a simple print of MongramFreq must give something like this: 

('atelier',): 17, ('430',): 17, ('jabli',): 17, ('mall',): 17, ('palmeraies',): 17, ('r4',): 17, ('dégagée',): 17, ('fatha',): 17

but output.txt file is not even created.

I corrected my code as follows:

import nltk, re, string, collections
from nltk.util import ngrams 
with open("titles.txt", "r", encoding='utf-8') as file:
text = file.read()
tokenized = text.split()
Threegrams = ngrams(tokenized, 3)
ThreegramFreq = collections.Counter(Threegrams)
for i in ThreegramFreq.elements():
with open('output.txt', 'a') as w:
w.write(str(i))
w.close()

Here is a sample of output.txt:

('les', 'étudiants', 'S1')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('Joli', 'appartement', 's3')('appartement', 's3', 'aux')('appartement', 's3', 'aux')('appartement', 's3', 'aux')('appartement', 's3', 'aux')('appartement', 's3', 'aux')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('s3', 'aux', 'jardins')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('aux', 'jardins', 'de')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')('jardins', 'de', 'carthage')

But I need to have the frequency of each 3-gram in my output.txt file. How to do ?

nltk nlp python
2 Answers

My suggestion would be to use CountVectorizer for this!

s='''Joli appartement s3 aux jardins de carthage mz823
Villa 600m2 haut standing à hammamet
Hammem lif
S2 manzah 7
Terrain constructible de 252m2 clôturé
Terrain nu a gammarth
Terrain agrecole al fahes
Bureau 17 pièces
Usine 5000m2 mannouba'''

text = s.split('\n')

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(text)

we have transformed input text into bag of words. we are going to count the frequency of each word, by making sum across rows. The word corresponding to each column names would be retrieved from vectorizer.get_feature_names().

MonogramFreq = {word:count for count,word in zip(X.sum(0).tolist()[0],
                                            vectorizer.get_feature_names())}
print(MonogramFreq)

output:

{'17': 1, '252m2': 1, '5000m2': 1, '600m2': 1, 'agrecole': 1, 'al': 1, 'appartement': 1, 'aux': 1, 'bureau': 1, 'carthage': 1, 'clôturé': 1, 'constructible': 1, 'de': 2, 'fahes': 1, 'gammarth': 1, 'hammamet': 1, 'hammem': 1, 'haut': 1, 'jardins': 1, 'joli': 1, 'lif': 1, 'mannouba': 1, 'manzah': 1, 'mz823': 1, 'nu': 1, 'pièces': 1, 's2': 1, 's3': 1, 'standing': 1, 'terrain': 3, 'usine': 1, 'villa': 1}

EDIT:

For getting more than mono-grams, such as bi-grams, tri-grams upto 6-grams.

vectorizer = CountVectorizer(ngram_range=(1,6))
user_63476290

TL;DR

But please at least read the comments:

from collections import Counter

from nltk import word_tokenize, ngrams

text='''Joli appartement s3 aux jardins de carthage mz823
Villa 600m2 haut standing à hammamet
Hammem lif
S2 manzah 7
Terrain constructible de 252m2 clôturé
Terrain nu a gammarth
Terrain agrecole al fahes
Bureau 17 pièces
Usine 5000m2 mannouba'''

# Create a counter object to track ngrams and counts.
ngram_counters = Counter()

# Split the text into sentences, 
# For now, assume '\n' delimits the sentences.
for line in text.split('\n'):
    # Update the counters with ngrams in each sentence,
    ngram_counters.update(ngrams(word_tokenize(line), n=3))

# Opens a file to print out.
with open('ngram_counts.tsv', 'w') as fout:
    # Iterate through the counter object, like a dictionary.
    for ng, counts in ngram_counters.items():
        # Use space to join the tokens in the ngrams before printing.
        # Print the counts in a separate column.
        print(' '.join(ng) +'\t' + str(counts), end='\n', file=fout)
user_6105690
Related