<commit_before>

import requests
import re
import matplotlib.pyplot as plt
import numpy as np
import redis
import utils

from nltk.stem import PorterStemmer
from collections import OrderedDict, Counter
from sklearn.neighbors import KernelDensity
from itertools import combinations


class Text(object):


    @classmethod
    def from_file(cls, path, slug, **kwargs):

        """
        Create a text from a filepath.

        :param path: The filepath.
        :param slug: A Redis key prefix.
        """

        # Clear the namespace.
        text = cls(slug, **kwargs)
        text.clear()

        # Index the text.
        text.index(open(path, 'r').read())
        return text


    def __init__(self, slug, **kwargs):

        """
        Set the key prefix, connect to Redis.

        :param slug: A Redis key prefix.
        """

        self.slug = slug

        # Redis connection.
        self.redis = redis.StrictRedis(**kwargs)

        # Porter stemmer.
        self.stem = PorterStemmer().stem


    @property
    def wordcount_key(self):

        """
        Redis key for the word counter.
        """

        return self.slug+':wordcount'


    @property
    def wordcount(self):

        """
        How many words are in the text?
        """

        return int(self.redis.get(self.wordcount_key))


    @property
    def text_key(self):

        """
        Redis key for the raw text string.
        """

        return self.slug+':text'


    @property
    def text(self):

        """
        Get the raw text string.
        """

        return self.redis.get(self.text_key)


    @property
    def terms_key(self):

        """
        Redis key for the set of unique terms.
        """

        return self.slug+':terms'


    @property
    def terms(self):

        """
        Get the set of unique terms.
        """

        return self.redis.smembers(self.terms_key)


    def has_term(self, term):

        """
        Does the text contain a term?

        :param term: A stemmed term.
        """

        return self.redis.sismember(self.terms_key, term)


    def token_key(self, offset):

        """
        Redis key for a token at a given offset.

        :param offset: An integer offset.
        """

        return self.slug+':token:'+str(offset)


    def token(self, offset):

        """
        Get the hash for a token at a given offset, casting the start and end
        character offsets to integers.

        :param offset: An integer offset.
        """

        token = self.redis.hgetall(self.token_key(offset))
        return utils.numberify_dict(token)


    def offsets_key(self, term):

        """
        Redis key for the set of term offsets.

        :param term: A stemmed term.
        """

        return self.slug+':offsets:'+term


    def offsets(self, term):

        """
        Get the offsets for a term as a sorted list of integers.

        :param term: A stemmed term.
        """

        offsets = self.redis.smembers(self.offsets_key(term))
        return sorted(list(map(int, offsets)))


    def index(self, text):

        """
        Tokenize the text.

        :param text: The text, as a raw string.
        """

        pipe = self.redis.pipeline()

        # Store the original text
        pipe.set(self.text_key, text)

        for i, token in enumerate(utils.tokenize(text)):

            term = token['stemmed']

            pipe.hmset(self.token_key(i), token)    # Token
            pipe.sadd(self.terms_key, term)         # Term
            pipe.sadd(self.offsets_key(term), i)    # Term -> offset
            pipe.incr(self.wordcount_key)           # Wordcount + 1

        pipe.execute()


    def clear(self):<commit_msg>Breaks apart separate setter methods.<commit_after>

import requests
import re
import matplotlib.pyplot as plt
import numpy as np
import redis
import utils

from nltk.stem import PorterStemmer
from collections import OrderedDict, Counter
from sklearn.neighbors import KernelDensity
from itertools import combinations


class Text(object):


    @classmethod
    def from_file(cls, path, slug, **kwargs):

        """
        Create a text from a filepath.

        :param path: The filepath.
        :param slug: A Redis key prefix.
        """

        # Clear the namespace.
        text = cls(slug, **kwargs)
        text.clear()

        # Index the text.
        text.index(open(path, 'r').read())
        return text


    def __init__(self, slug, **kwargs):

        """
        Set the key prefix, connect to Redis.

        :param slug: A Redis key prefix.
        """

        self.slug = slug

        # Redis connection and pipeline.
        self.redis = redis.StrictRedis(**kwargs)
        self.pipeline = self.redis.pipeline()

        # Porter stemmer.
        self.stem = PorterStemmer().stem


    @property
    def wordcount_key(self):

        """
        Redis key for the word counter.
        """

        return self.slug+':wordcount'


    def inc_wordcount(self):

        """
        Increment the wordcount.
        """

        self.pipeline.incr(self.wordcount_key)


    @property
    def wordcount(self):

        """
        How many words are in the text?
        """

        return int(self.redis.get(self.wordcount_key))


    @property
    def text_key(self):

        """
        Redis key for the raw text string.
        """

        return self.slug+':text'


    @property
    def text(self):

        """
        Get the raw text string.
        """

        return self.redis.get(self.text_key)


    @property
    def terms_key(self):

        """
        Redis key for the set of unique terms.
        """

        return self.slug+':terms'


    def set_term(self, term):

        """
        Register the existence of a term.

        :param term: A stemmed term.
        """

        self.pipeline.sadd(self.terms_key, term)


    @property
    def terms(self):

        """
        Get the set of unique terms.
        """

        return self.redis.smembers(self.terms_key)


    def has_term(self, term):

        """
        Does the text contain a term?

        :param term: A stemmed term.
        """

        return self.redis.sismember(self.terms_key, term)


    def token_key(self, offset):

        """
        Redis key for a token at a given offset.

        :param offset: An integer offset.
        """

        return self.slug+':token:'+str(offset)


    def set_token(self, offset, token):

        """
        Index a token.

        :param offset: The integer offset of the token.
        :param token: The token dictionary.
        """

        self.pipeline.hmset(self.token_key(offset), token)


    def token(self, offset):

        """
        Get the hash for a token at a given offset, casting the start and end
        character offsets to integers.

        :param offset: An integer offset.
        """

        token = self.redis.hgetall(self.token_key(offset))
        return utils.numberify_dict(token)


    def offsets_key(self, term):

        """
        Redis key for the set of term offsets.

        :param term: A stemmed term.
        """

        return self.slug+':offsets:'+term


    def set_offset(self, offset, term):

        """
        Register a term instance offset.

        :param offset: The integer offset of the instance.
        :param term: The stemmed term.
        """

        self.pipeline.sadd(self.offsets_key(term), offset)


    def offsets(self, term):

        """
        Get the offsets for a term as a sorted list of integers.

        :param term: A stemmed term.
        """

        offsets = self.redis.smembers(self.offsets_key(term))
        return sorted(list(map(int, offsets)))


    def distance_key(self, term1, term2):

        """
        Redis key for the density distance between tero terms.

        :param term1: The first term.
        :param term2: The second term.
        """

        terms = '_'.join(sorted((term1, term2)))
        return self.slug+':distance:'+terms


    def index(self, text):

        """
        Tokenize the text.

        :param text: The text, as a raw string.
        """

        # Store the original text
        self.pipeline.set(self.text_key, text)

        # Index tokens, terms, and wordcount.
        for i, token in enumerate(utils.tokenize(text)):
            term = token['stemmed']
            self.set_token(i, token)
            self.set_term(term)
            self.set_offset(i, term)
            self.inc_wordcount()

        self.pipeline.execute()


    def clear(self):
{'fraction_non_alphanumeric': 0.10260889820638248, 'fraction_numerical': 0.000815280689494526, 'mean_word_length': 1.8007175472928898, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '301534', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2448, 'n_words': 745}