<commit_before>'''
tests.py: part of expfactory package
tests for experiments and batteries, not for expfactory-python

'''
from selenium.common.exceptions import WebDriverException, UnexpectedAlertPresentException
from expfactory.experiment import validate, get_experiments, load_experiment
from expfactory.views import generate_experiment_web, tmp_experiment
from numpy.testing import assert_equal, assert_string_equal
from selenium.webdriver.common.keys import Keys
from expfactory.utils import find_directories, get_url
from numpy.random import choice
from selenium import webdriver
from threading import Thread
import SimpleHTTPServer
from time import sleep
import SocketServer
import webbrowser
import requests
import shutil
import numpy
import json
import re
import sys
import os

# subclass SimpleHTTPServer to capture error messages
class ExpfactoryServer(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def log_message(self, format, *args):
        sys.stderr.write("%s - - [%s] %s\n" %
                     (self.address_string(),
                      self.log_date_time_string(),
                      format%args))
        # This is a workaround for jspsych error trying to GET html
        if not re.search("div",format%args) and not re.search("function",format%args):
            if re.search("404",format%args):
                # closing server will trigger error
                raise IOError(format%args)
    def log_error(self, format, *args):
        # We will catch errors in log_messages
        pass

def validate_experiment_directories(experiment_folder):
    experiments = find_directories(experiment_folder)
    for contender in experiments:
        assert_equal(validate(contender),True)

def validate_experiment_tag(experiment_folder):
    '''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
    '''
    experiments = find_directories(experiment_folder)
    print "Testing %s experiment for definition of exp_id in experiment.js..."
    for contender in experiments:
        if validate(contender,warning=False) == True:
            experiment = load_experiment(contender)
            tag = experiment[0]["exp_id"]

            # Experiment MUST contain experiment.js to run main experiment
            print "TESTING %s for exp_id in experiment.js..." %tag
            assert_equal("experiment.js" in experiment[0]["run"],True)

            if "experiment.js" in experiment[0]["run"]:
                experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r") 
                experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
                experiment_js_file.close()
                experiment_js = "".join(experiment_js_list)
                [x]
                has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None 
                assert_equal(has_exp_id,True)
                # Ensure all are formatted correctly
                exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
                for exp_id_instance in exp_id_instances:
                    assert_equal(re.search(tag,exp_id_instance[0])!=None,True) 

def validate_circle_yml(experiment_repo):
    '''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml
    :param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files
    '''
    if "CIRCLE_BRANCH" in os.environ.keys():
        experiments = get_experiments(experiment_repo,load=True,warning=False)<commit_msg>adding line number to help with debugging<commit_after>'''
tests.py: part of expfactory package
tests for experiments and batteries, not for expfactory-python

'''
from selenium.common.exceptions import WebDriverException, UnexpectedAlertPresentException
from expfactory.experiment import validate, get_experiments, load_experiment
from expfactory.views import generate_experiment_web, tmp_experiment
from numpy.testing import assert_equal, assert_string_equal
from selenium.webdriver.common.keys import Keys
from expfactory.utils import find_directories, get_url
from numpy.random import choice
from selenium import webdriver
from threading import Thread
import SimpleHTTPServer
from time import sleep
import SocketServer
import webbrowser
import requests
import shutil
import numpy
import json
import re
import sys
import os

# subclass SimpleHTTPServer to capture error messages
class ExpfactoryServer(SimpleHTTPServer.SimpleHTTPRequestHandler):
    def log_message(self, format, *args):
        sys.stderr.write("%s - - [%s] %s\n" %
                     (self.address_string(),
                      self.log_date_time_string(),
                      format%args))
        # This is a workaround for jspsych error trying to GET html
        if not re.search("div",format%args) and not re.search("function",format%args):
            if re.search("404",format%args):
                # closing server will trigger error
                raise IOError(format%args)
    def log_error(self, format, *args):
        # We will catch errors in log_messages
        pass

def validate_experiment_directories(experiment_folder):
    experiments = find_directories(experiment_folder)
    for contender in experiments:
        assert_equal(validate(contender),True)

def validate_experiment_tag(experiment_folder):
    '''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
    '''
    experiments = find_directories(experiment_folder)
    print "Testing %s experiment for definition of exp_id in experiment.js..."
    for contender in experiments:
        if validate(contender,warning=False) == True:
            experiment = load_experiment(contender)
            tag = experiment[0]["exp_id"]

            # Experiment MUST contain experiment.js to run main experiment
            print "TESTING %s for exp_id in experiment.js..." %tag
            assert_equal("experiment.js" in experiment[0]["run"],True)

            if "experiment.js" in experiment[0]["run"]:
                experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r") 
                experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
                experiment_js_file.close()
                experiment_js = "".join(experiment_js_list)
                [x]
                has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None 
                assert_equal(has_exp_id,True)
                # Ensure all are formatted correctly
                exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
                line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
                for e in range(len(exp_id_instances)):
                    exp_id_instance = exp_id_instances[e]
                    line_number = line_numbers[e]
                    print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
                    assert_equal(re.search(tag,exp_id_instance[0])!=None,True) 

def validate_circle_yml(experiment_repo):
    '''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml
    :param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files
    '''
    if "CIRCLE_BRANCH" in os.environ.keys():
        experiments = get_experiments(experiment_repo,load=True,warning=False)
{'fraction_non_alphanumeric': 0.09380793474230627, 'fraction_numerical': 0.0023482882214806576, 'mean_word_length': 3.350537634408602, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1414148', 'n_tokens_mistral': 2284, 'n_tokens_neox': 2208, 'n_words': 629}