<commit_before>from redis import Redis, RedisError, ConnectionPool
import itertools
import time
from .document import Document
from .result import Result
from .query import Query, Filter


class Field(object):

    NUMERIC = 'NUMERIC'
    TEXT = 'TEXT'
    WEIGHT = 'WEIGHT'
    GEO = 'GEO'
    SORTABLE = 'SORTABLE'

    def __init__(self, name, *args):
        self.name = name
        self.args = args
    
    def redis_args(self):

        return [self.name] + list(self.args)


class TextField(Field):
    """
    TextField is used to define a text field in a schema definition
    """

    def __init__(self, name, weight=1.0, sortable = False):
        args = [Field.TEXT, Field.WEIGHT, weight]
        if sortable:
            args.append(Field.SORTABLE)

        Field.__init__(self, name, *args)

<commit_msg>Add `no_stem` option to TextField<commit_after>from redis import Redis, RedisError, ConnectionPool
import itertools
import time
from .document import Document
from .result import Result
from .query import Query, Filter


class Field(object):

    NUMERIC = 'NUMERIC'
    TEXT = 'TEXT'
    WEIGHT = 'WEIGHT'
    GEO = 'GEO'
    SORTABLE = 'SORTABLE'

    def __init__(self, name, *args):
        self.name = name
        self.args = args
    
    def redis_args(self):

        return [self.name] + list(self.args)


class TextField(Field):
    """
    TextField is used to define a text field in a schema definition
    """
    NOSTEM = 'NOSTEM'

    def __init__(self, name, weight=1.0, sortable=False, no_stem=False):
        args = [Field.TEXT, Field.WEIGHT, weight]
        if sortable:
            args.append(Field.SORTABLE)
        if no_stem:
            args.append(self.NOSTEM)

        Field.__init__(self, name, *args)


{'fraction_non_alphanumeric': 0.10159817351598173, 'fraction_numerical': 0.00228310502283105, 'mean_word_length': 2.652083333333333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846687', 'n_tokens_mistral': 570, 'n_tokens_neox': 551, 'n_words': 165}