most frequently occurring words of a text file excluding stopwords

user_13183562 0
Viewed 1166

I have a text file in french that I want to count its most occurring words, without taking in consideration stop words. Here's the code: 

with open('./text_file.txt', 'r', encoding='utf8') as f:
    s = f.read()

num_chars = len(s)
num_lines = s.count('\n')

#call split with no arguments
words = s.split()
d = {}
for w in words:
    if w in d:
        d[w] += 1
    else:
        d[w] = 1

num_words = sum(d[w] for w in d)

lst = [(d[w],w) for w in d]
lst.sort()
lst.reverse()

# nltk treatment
from nltk.corpus import stopwords # Import the stop word list
from nltk.tokenize import wordpunct_tokenize

stop_words = set(stopwords.words('french')) # creating a set makes the searching faster
print (stop_words)
print ([word for word in lst if word not in stop_words])


print('\n The 50 most frequent words are /n')

i = 1
for count, word in lst[:50]:
    print('%2s. %4s %s' %(i,count,word))
    i+= 1

This returns the most occurred words including the stop words. Any better idea how to do this?

nltk python
4 Answers

Here is a simplified version:

from nltk.corpus import stopwords # Import the stop word list
from nltk.tokenize import wordpunct_tokenize

with open('./text_file.txt', 'r', encoding='utf8') as f:
    words = f.read().split()

d = {}
stop_words = set(stopwords.words('french')) # creating a set makes the searching faster
for w in words:
    if w not in stop_words:
        if w in d:
            d[w] += 1
        else:
            d[w] = 1

lst = sorted([(d[w],w) for w in d],reverse=True)
print (stop_words)
print ([word for word in lst if word not in stop_words])
print('\n The 50 most frequent words are /n')

i = 1
for count, word in lst[:50]:
    print('%2s. %4s %s' %(i,count,word))
    i += 1
user_135524700 
with open("/yourFile.txt", "r") as file:
    words = file.read().split()

    cptwords = {}

    for word in words:
        if word[-1] in [",", ".", "\n", ":", "!", "?", ";"]:
            word.rstrip()

        cptwords.setdefault(word, 0)
        cptwords[word] += 1

    cptwords = sorted(cptwords.items(), key = lambda x: x[1], reverse = True)

    print(f"The first 50 most used words are {[truc[0] for truc in cptwords[:50]]}")

Thats an easy way to do this.

user_125447990

Here is an even cleaner (and probably faster) solution using collections.Counter:

from collections import Counter
from nltk.corpus import stopwords # Import the stop word list
from nltk.tokenize import wordpunct_tokenize
NUM_WORDS = 50

with open('./text_file.txt', 'r', encoding='utf8') as f:
    words = f.read().split()

word_counts = Counter(word for word in words
                      if word not in set(stopwords.words('french')))
print(f'\nThe {NUM_WORDS} most frequent words are:\n')
for i, (word, count) in enumerate(word_counts.most_common(NUM_WORDS)):
    print('%2s. %4s %s' % (i, count, word))
user_92487930

NLTK has a class for counting frequencies called FreqDist with bunch of handy methods available. You can use it as follows:

from nltk.tokenize import wordpunct_tokenize
from nltk.probability import FreqDist
from nltk.corpus import stopwords


with open('text_file.txt', 'r', encoding='utf8') as f:
    text = f.read()

fd = FreqDist(
    word
    for word in wordpunct_tokenize(text)
    if word not in set(stopwords.words('french'))
)
fd.pprint()
user_74499670
Related