<commit_before>'''
corpus.py

part of the wordfish python package: extracting relationships of terms from corpus
this set of functions works with different plugins (in plugins folder) to produce input corpus
for parsing.

'''

from textblob import TextBlob

def save_sentences(articles,output_dir="."):
    '''save_sentences: 
    parses a dictionary with articles and sentences into output sentence files
    Parameters
    ==========
    articles: list or dict
        a list or dictionary of articles to parse and save to sentences.txt files.
        If you have a meaningful unique ID, provide a dictionary with the UID as key
        If not, provide a list and an integer will be generated instead.
    output_dir: path
        full path that corresponds to the plugins specific directory under "corpus"
        eg $WORK/corpus/neurosynth
    '''
    if isinstance(articles,dict):
        for uid, text in articles.iteritems():
            save_sentences_single(uid,text,output_dir)

    elif isinstance(articles,list):
        for uid in range(len(articles)):
            save_sentences_single(uid,articles[uid],output_dir)

def save_sentences_single(uid,text,output_dir,remove_non_english_chars=True):
    '''save_sentence_single
    Write sentences to file

    88390|"<text><p>sentence1</p><p>sentence2</p><p></text>"
    88391|"<text><p>sentence1</p><p>sentence2</p><p></text>"
    We should use utf-8 http://www.postgresql.org/docs/9.0/static/multibyte.html
 
    Parameters
    ==========
    uid: int or string
        a unique ID for the article
    text: str
        raw text of the article    
    output_dir: path
        full path to a plugins corpus directory
    '''
    output_file = "%s_sentences.txt" %(output_dir)
    filey = open(output_file,"wb")
    filey.write('%s|"<text>' %uid)
    blob = TextBlob(text)
    for sentence in blob.sentences:
        sentence = '<p>%s</p>' %sentence.raw.replace("\t","").replace("|"," ").replace("\n","").replace("\r","")
        filey.write(sentence.replace("}","").replace("{","").encode("utf-8"))
    filey.write('</text>"\n')
    filey.close()<commit_msg>	modified:   wordfish/corpus.py<commit_after>'''
corpus.py

part of the wordfish python package: extracting relationships of terms from corpus
this set of functions works with different plugins (in plugins folder) to produce input corpus
for parsing.

'''

from textblob import TextBlob

def save_sentences(articles,output_dir="."):
    '''save_sentences: 
    parses a dictionary with articles and sentences into output sentence files
    Parameters
    ==========
    articles: list or dict
        a list or dictionary of articles to parse and save to sentences.txt files.
        If you have a meaningful unique ID, provide a dictionary with the UID as key
        If not, provide a list and an integer will be generated instead.
    output_dir: path
        full path that corresponds to the plugins specific directory under "corpus"
        eg $WORK/corpus/neurosynth
    '''
    if isinstance(articles,dict):
        for uid, text in articles.iteritems():
            save_sentences_single(uid,text,output_dir)

    elif isinstance(articles,list):
        for uid in range(len(articles)):
            save_sentences_single(uid,articles[uid],output_dir)

def save_sentences_single(uid,text,output_dir,remove_non_english_chars=True):
    '''save_sentence_single
    Write sentences to file

    88390|"<text><p>sentence1</p><p>sentence2</p><p></text>"
    88391|"<text><p>sentence1</p><p>sentence2</p><p></text>"
    We should use utf-8 http://www.postgresql.org/docs/9.0/static/multibyte.html
 
    Parameters
    ==========
    uid: int or string
        a unique ID for the article
    text: str
        raw text of the article    
    output_dir: path
        full path to a plugins corpus directory
    '''
    output_file = "%s/sentences.txt" %(output_dir)
    filey = open(output_file,"wb")
    filey.write('%s|"<text>' %uid)
    blob = TextBlob(text)
    for sentence in blob.sentences:
        sentence = '<p>%s</p>' %sentence.raw.replace("\t","").replace("|"," ").replace("\n","").replace("\r","")
        filey.write(sentence.replace("}","").replace("{","").encode("utf-8"))
    filey.write('</text>"\n')
    filey.close()
{'fraction_non_alphanumeric': 0.12963832785345233, 'fraction_numerical': 0.008454673555659934, 'mean_word_length': 3.3816872427983538, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3904598', 'n_tokens_mistral': 1313, 'n_tokens_neox': 1244, 'n_words': 371}