def ngrams(n, f, prune_after=10000):
    counter = collections.Counter()
deque = collections.deque(maxlen=n)
for i, line in enumerate(f):
    deque.clear()
words = ['<s>'] + line.split() + ['</s>']
deque.extend(words[:n - 1])
for word in words[n - 1:]:
    deque.append(word)
ngram = tuple(str(w) for w in deque)
if i < prune_after or ngram in counter:
    counter[ngram] += 1
return counter
Эффективная экстракция 1-5 грамм с помощью питона