<commit_before>import os
import nltk
import itertools
import codecs

CURDIR = os.path.dirname(os.path.realpath(__file__))


def load_nips(years=None, raw=False):
    # load data
    if not years:
        years = xrange(2008, 2015)
    files = ['nips-{}.dat'.format(year)
             for year in years]

    docs = []
    for f in files:
        with codecs.open('{}/datasets/{}'.format(CURDIR, f), 'r', 'utf8') as f:
            for l in f:
                if raw:
                    docs.append(l.strip())
                else:
                    sents = nltk.sent_tokenize(l.strip().lower())
                    docs.append(list(itertools.chain(*map(
                        nltk.word_tokenize, sents))))
    return docs


def load_lemur_stopwords():
    with codecs.open('datasets/lemur-stopwords.txt', 
                     'r' 'utf8') as f:
        return map(lambda s: s.strip(),
                   f.readlines())<commit_msg>added pos processing and result pickling for nips<commit_after>import os
import nltk
import itertools
import codecs
from toolz.functoolz import compose
import cPickle as pickle

CURDIR = os.path.dirname(os.path.realpath(__file__))


def load_nips(years=None, raw=False):
    # load data
    if not years:
        years = xrange(2008, 2015)
    files = ['nips-{}.dat'.format(year)
             for year in years]

    docs = []
    for f in files:
        with codecs.open('{}/datasets/{}'.format(CURDIR, f), 'r', 'utf8') as f:
            for l in f:
                if raw:
                    docs.append(l.strip())
                else:
                    sents = nltk.sent_tokenize(l.strip().lower())
                    docs.append(list(itertools.chain(*map(
                        nltk.word_tokenize, sents))))
    return docs


def tag_nips_and_pickle(years=None):
    """
    pos tag the nips collection of given years and then pickle them
    """
    # load data
    if not years:
        years = xrange(2008, 2015)
    files = ['nips-{}.dat'.format(year)
             for year in years]

    for year, f in zip(years, files):

        print "processing year {}...".format(year)

        with codecs.open('{}/datasets/{}'.format(CURDIR, f), 'r', 'utf8') as f:
            docs = []
            for l in f:
                sents = nltk.sent_tokenize(l.strip().lower())
                docs.append(list(itertools.chain(
                    *map(
                        compose(nltk.pos_tag, nltk.word_tokenize),
                        sents))))

            pickle.dump(docs,
                        open('{}/datasets/nips-pos-{}.pkl'.format(CURDIR,
                                                                  year),
                             'w'))
                

def load_tagged_nips(years=None):
    if not years:
        years = xrange(2008, 2015)
        files = ['nips-pos-{}.pkl'.format(year)
                 for year in years]
    docs = []
    for f in files:
        full_path = '{}/datasets/{}'.format(CURDIR, f)
        docs += pickle.load(open(full_path))
    return docs


def load_lemur_stopwords():
    with codecs.open('datasets/lemur-stopwords.txt', 
                     'r' 'utf8') as f:
        return map(lambda s: s.strip(),
                   f.readlines())


if __name__ == "__main__":
    tag_nips_and_pickle()
{'fraction_non_alphanumeric': 0.126328575766778, 'fraction_numerical': 0.011235955056179775, 'mean_word_length': 1.6415396952686447, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7272101', 'n_tokens_mistral': 1086, 'n_tokens_neox': 1001, 'n_words': 239}