<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""Automatically detect common phrases -- multi-word expressions / word n-grams -- from a stream of sentences.

Inspired by:

* `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality"
  <https://arxiv.org/abs/1310.4546>`_
* `"Normalized (Pointwise) Mutual Information in Colocation Extraction" by <NAME>
  <https://svn.spraakdata.gu.se/repos/gerlof/pub/www/Docs/npmi-pfd.pdf>`_


Examples
--------
.. sourcecode:: pycon

    >>> from gensim.test.utils import datapath
    >>> from gensim.models.word2vec import Text8Corpus
    >>> from gensim.models.phrases import Phrases, Phraser
    >>>
    >>> sentences = Text8Corpus(datapath('testcorpus.txt'))
    >>> phrases = Phrases(sentences, min_count=1, threshold=1)  # train model
    >>> phrases[[u'trees', u'graph', u'minors']]  # apply model to sentence
    [u'trees_graph', u'minors']
    >>>
    >>> phrases.add_vocab([["hello", "world"], ["meow"]])  # update model with new sentences
    >>>
    >>> bigram = Phraser(phrases)  # construct faster model (this is only an wrapper)
    >>> bigram[[u'trees', u'graph', u'minors']]  # apply model to sentence
    [u'trees_graph', u'minors']
    >>>
    >>> for sent in bigram[sentences]:  # apply model to text corpus
    ...     pass

"""

import sys
import os<commit_msg>Fix usage example in phrases.py (#2575)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2011 <NAME> <<EMAIL>>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""
Automatically detect common phrases -- aka multi-word expressions, word n-gram collocations -- from
a stream of sentences.

Inspired by:

* `Mikolov, et. al: "Distributed Representations of Words and Phrases and their Compositionality"
  <https://arxiv.org/abs/1310.4546>`_
* `"Normalized (Pointwise) Mutual Information in Colocation Extraction" by <NAME>
  <https://svn.spraakdata.gu.se/repos/gerlof/pub/www/Docs/npmi-pfd.pdf>`_


Examples
--------
.. sourcecode:: pycon

    >>> from gensim.test.utils import datapath
    >>> from gensim.models.word2vec import Text8Corpus
    >>> from gensim.models.phrases import Phrases, Phraser
    >>>
    >>> # Load training data.
    >>> sentences = Text8Corpus(datapath('testcorpus.txt'))
    >>> # The training corpus must be a sequence (stream, generator) of sentences,
    >>> # with each sentence a list of tokens:
    >>> print(list(sentences)[0][:10])
    ['computer', 'human', 'interface', 'computer', 'response', 'survey', 'system', 'time', 'user', 'interface']
    >>>
    >>> # Train a toy bigram model.
    >>> phrases = Phrases(sentences, min_count=1, threshold=1)
    >>> # Apply the trained phrases model to a new, unseen sentence.
    >>> phrases[['trees', 'graph', 'minors']]
    ['trees_graph', 'minors']
    >>> # The toy model considered "trees graph" a single phrase => joined the two
    >>> # tokens into a single token, `trees_graph`.
    >>>
    >>> # Update the model with two new sentences on the fly.
    >>> phrases.add_vocab([["hello", "world"], ["meow"]])
    >>>
    >>> # Export the trained model = use less RAM, faster processing. Model updates no longer possible.
    >>> bigram = Phraser(phrases)
    >>> bigram[['trees', 'graph', 'minors']]  # apply the exported model to a sentence
    ['trees_graph', 'minors']
    >>>
    >>> # Apply the exported model to each sentence of a corpus:
    >>> for sent in bigram[sentences]:
    ...     pass
    >>>
    >>> # Save / load an exported collocation model.
    >>> bigram.save("/tmp/my_bigram_model.pkl")
    >>> bigram_reloaded = Phraser.load("/tmp/my_bigram_model.pkl")
    >>> bigram_reloaded[['trees', 'graph', 'minors']]  # apply the exported model to a sentence
    ['trees_graph', 'minors']

"""

import sys
import os
{'fraction_non_alphanumeric': 0.16253513927932534, 'fraction_numerical': 0.01098901098901099, 'mean_word_length': 3.5511627906976746, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 148, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1273905', 'n_tokens_mistral': 1329, 'n_tokens_neox': 1259, 'n_words': 356}