<commit_before>import sys, ast
from StrongCNN.IO.config_parser import parse_configfile
from StrongCNN.IO.load_images import load_data
from StrongCNN.IO.augment_data import augment_methods, augment_data
from StrongCNN.pipeline.build_pipeline import build_pipeline
from _tools import train_model, dump_model
import time

cfgdir = sys.argv[1]
cfg = parse_configfile(cfgdir)

start_time = time.time()

# Collect training data
X_train, y_train, _ = load_data(cfg['train_filenames']['non_lens_glob'], 
                                cfg['train_filenames']['lens_glob'])


if 'augment_train_data' in cfg.keys() :
    X_train, y_train = augment_data( X_train, y_train, 
                                     cfg['augment_train_data']['method_label'],
                                     **ast.literal_eval(cfg['augment_train_data']['method_kwargs']))  
    
print "len(X_train) =", len(X_train)
print "len(y_train) =", len(y_train)

print X_train[0].shape


print "train glob ", cfg['train_filenames']['non_lens_glob'], cfg['train_filenames']['lens_glob']

# Build the pipeline<commit_msg>changed StrongCNN to allpipeline in generate<commit_after>import sys, ast
from allpipeline.IO.config_parser import parse_configfile
from allpipeline.IO.load_images import load_data
from allpipeline.IO.augment_data import augment_methods, augment_data
from allpipeline.pipeline.build_pipeline import build_pipeline
from _tools import train_model, dump_model
import time

cfgdir = sys.argv[1]
cfg = parse_configfile(cfgdir)

start_time = time.time()

# Collect training data
X_train, y_train, _ = load_data(cfg['train_filenames']['non_lens_glob'], 
                                cfg['train_filenames']['lens_glob'])


if 'augment_train_data' in cfg.keys() :
    X_train, y_train = augment_data( X_train, y_train, 
                                     cfg['augment_train_data']['method_label'],
                                     **ast.literal_eval(cfg['augment_train_data']['method_kwargs']))  
    
print "len(X_train) =", len(X_train)
print "len(y_train) =", len(y_train)

print "train glob ", cfg['train_filenames']['non_lens_glob'], cfg['train_filenames']['lens_glob']

# Build the pipeline
{'fraction_non_alphanumeric': 0.11065197428833792, 'fraction_numerical': 0.0013774104683195593, 'mean_word_length': 3.9635535307517085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7239253', 'n_tokens_mistral': 694, 'n_tokens_neox': 678, 'n_words': 136}