<commit_before>

import utils

from collections import OrderedDict
from text import Text
from matrix import Matrix
from graphs import *


def frequent(path, term_depth=500, skim_depth=10, d_weights=False, **kwargs):

    """
    Use most frequent terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    print 'Indexing terms:'
    m.index(t.most_frequent_terms(term_depth), **kwargs)

    g = Skimmer()

    print 'Generating graph:'
    g.build(m, skim_depth, d_weights)

    return g


def clumpy(path, term_depth=500, skim_depth=10, d_weights=False, **kwargs):

    """
    Use "clumpiest" terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    print 'Indexing terms:'
    m.index(t.densities(**kwargs).keys()[:term_depth], **kwargs)

    g = Skimmer()

    print 'Generating graph:'
    g.build(m, skim_depth, d_weights)

    return g


def listserv(path, term_depth=3000, spike_depth=1000, skim_depth=6,
             spike_bandwidth=500000, **kwargs):

    """
    Use most "spiky" terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    # Get the top X most frequent terms.
    frequent = t.most_frequent_terms(term_depth)<commit_msg>Using a higher default skim depth on the listserv builder.<commit_after>

import utils

from collections import OrderedDict
from text import Text
from matrix import Matrix
from graphs import *


def frequent(path, term_depth=500, skim_depth=10, d_weights=False, **kwargs):

    """
    Use most frequent terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    print 'Indexing terms:'
    m.index(t.most_frequent_terms(term_depth), **kwargs)

    g = Skimmer()

    print 'Generating graph:'
    g.build(m, skim_depth, d_weights)

    return g


def clumpy(path, term_depth=500, skim_depth=10, d_weights=False, **kwargs):

    """
    Use "clumpiest" terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    print 'Indexing terms:'
    m.index(t.densities(**kwargs).keys()[:term_depth], **kwargs)

    g = Skimmer()

    print 'Generating graph:'
    g.build(m, skim_depth, d_weights)

    return g


def listserv(path, term_depth=3000, spike_depth=1000, skim_depth=30,
             spike_bandwidth=500000, **kwargs):

    """
    Use most "spiky" terms.
    """

    t = Text.from_file(path)
    m = Matrix(t)

    # Get the top X most frequent terms.
    frequent = t.most_frequent_terms(term_depth)
{'fraction_non_alphanumeric': 0.12375, 'fraction_numerical': 0.02125, 'mean_word_length': 3.1756521739130434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3787113', 'n_tokens_mistral': 899, 'n_tokens_neox': 807, 'n_words': 206}