<commit_before>"""N-gram experiment."""

# for as long as it's not yet pip installable
import sys
sys.path.append('/home/emmery/git/shed/')
# -----

from shed.experiment import Experiment
from shed.featurizer import Ngrams


conf = {
    "gram_experiment": {
        "name": "gram_experiment",
        "train_data": ["./n_gram.csv"],
        "has_header": True,
        "features": [Ngrams(level='char', n_list=[3])],
        "text_column": 1,
        "label_column": 0,
        "save": ("log")
    }
}

for f, c in conf.items():<commit_msg>updated n_gram exampe<commit_after>"""N-gram experiment."""

# for as long as it's not yet pip installable
import sys
sys.path.append('/dir/to/shed/')
# -----

try:
    from shed.experiment import Experiment
    from shed.featurizer import Ngrams
except ImportError:
    print("Could not load shed. Please update the path in this file.")

conf = {
    "gram_experiment": {
        "name": "gram_experiment",
        "train_data": ["./n_gram.csv"],
        "has_header": True,
        "features": [Ngrams(level='char', n_list=[3])],
        "text_column": 1,
        "label_column": 0,
        "save": ("log")
    }
}

for f, c in conf.items():
{'fraction_non_alphanumeric': 0.16807432432432431, 'fraction_numerical': 0.005067567567567568, 'mean_word_length': 2.8349514563106797, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781817', 'n_tokens_mistral': 410, 'n_tokens_neox': 396, 'n_words': 103}