<commit_before>import json
import random
from io import BytesIO
from itertools import chain

from nose.tools import eq_

from ...features import Feature
from ...languages import Language
from ..scorer import MLScorerModel
from ..svc import LinearSVCModel, RBFSVCModel, SVCModel


def process_float(): return float()
some_float = Feature("some_float", process_float(),
                      depends_on=[], returns=float)

def process_other_float(): return float()
other_float = Feature("other_float", process_other_float(),
                      depends_on=[], returns=float)
FEATURES = [some_float, other_float]


def is_badword(word): return word == "badword"
def is_misspelled(word): return word == "misspelled"
some_language = Language(is_badword, is_misspelled)
LANGUAGE = some_language

def train_score(model):
    deterministic = random.Random(0)
    observations = list(chain(
        zip(((some, other) for some, other in
             zip((deterministic.normalvariate(1, .3) for i in range(200)),
                 (deterministic.normalvariate(2, .5) for i in range(200)))),
            (True for i in range(200))),
        zip(((some, other) for some, other in
             zip((deterministic.normalvariate(-1, .5) for i in range(35)),
                 (deterministic.normalvariate(-2, .3) for i in range(35)))),
            (False for i in range(35)))
    ))
    deterministic.shuffle(observations)
    
    mid = int(len(observations)/2)
    train_set = observations[:mid]
    test_set = observations[mid:]
    
    model.train(train_set)
    score_doc = next(model.score([(-1,-2)]))
    
    eq_(score_doc['prediction'], False)
    print(score_doc)
    print(type(list(score_doc['probability'].keys())[0]))
    json.dumps(score_doc) # Checks if the doc is JSONable
    
    test_stats = model.test(test_set)
    
    del test_stats['roc']
    print(test_stats)
    assert test_stats['auc'] > 0.5

def pickle_and_unpickle(model):
    f = BytesIO()
    model.dump(f)
    f.seek(0) # Rewind the file
    reconstructed_model = MLScorerModel.load(f)
    eq_([f.name for f in reconstructed_model.features],
        [f.name for f in model.features])
    eq_(type(reconstructed_model), type(model))

def test_svc():
    model = SVCModel(FEATURES, LANGUAGE)
    train_score(model)
    pickle_and_unpickle(model)<commit_msg>Removes unnecessary print statements from test_svc<commit_after>import json
import random
from io import BytesIO
from itertools import chain

from nose.tools import eq_

from ...features import Feature
from ...languages import Language
from ..scorer import MLScorerModel
from ..svc import LinearSVCModel, RBFSVCModel, SVCModel


def process_float(): return float()
some_float = Feature("some_float", process_float(),
                      depends_on=[], returns=float)

def process_other_float(): return float()
other_float = Feature("other_float", process_other_float(),
                      depends_on=[], returns=float)
FEATURES = [some_float, other_float]


def is_badword(word): return word == "badword"
def is_misspelled(word): return word == "misspelled"
some_language = Language(is_badword, is_misspelled)
LANGUAGE = some_language

def train_score(model):
    deterministic = random.Random(0)
    observations = list(chain(
        zip(((some, other) for some, other in
             zip((deterministic.normalvariate(1, .3) for i in range(200)),
                 (deterministic.normalvariate(2, .5) for i in range(200)))),
            (True for i in range(200))),
        zip(((some, other) for some, other in
             zip((deterministic.normalvariate(-1, .5) for i in range(35)),
                 (deterministic.normalvariate(-2, .3) for i in range(35)))),
            (False for i in range(35)))
    ))
    deterministic.shuffle(observations)
    
    mid = int(len(observations)/2)
    train_set = observations[:mid]
    test_set = observations[mid:]
    
    model.train(train_set)
    score_doc = next(model.score([(-1,-2)]))
    
    eq_(score_doc['prediction'], False)
    json.dumps(score_doc) # Checks if the doc is JSONable
    
    test_stats = model.test(test_set)
    
    del test_stats['roc']
    print(test_stats)
    assert test_stats['auc'] > 0.5

def pickle_and_unpickle(model):
    f = BytesIO()
    model.dump(f)
    f.seek(0) # Rewind the file
    reconstructed_model = MLScorerModel.load(f)
    eq_([f.name for f in reconstructed_model.features],
        [f.name for f in model.features])
    eq_(type(reconstructed_model), type(model))

def test_svc():
    model = SVCModel(FEATURES, LANGUAGE)
    train_score(model)
    pickle_and_unpickle(model)
{'fraction_non_alphanumeric': 0.119140625, 'fraction_numerical': 0.013237847222222222, 'mean_word_length': 3.360454115421003, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4806608', 'n_tokens_mistral': 1514, 'n_tokens_neox': 1480, 'n_words': 364}