<commit_before>import cProfile
import random
import time
import graphs
import attractors
from shutil import copyfile
# from pebble import ProcessPool
from collections import namedtuple
import csv
import stochastic
import multiprocessing
import itertools
import numpy
import os
import sys
import signal, os, errno
from functools import wraps
import stat
import platform

attractor_estimation_n_iter = 100
node_measurement_prob = 0.8
n_bio_experiments = 50
relax_experiments = False
timeout_seconds = int(0.8 * 60 * 60)
graph_parent_dir = "cellcollective_models"
max_attractor_length = 5
graph_size_filter = 50
tests_per_graph = 20
limit_experiments_to_short_attractors = False
allow_suboptimal = True


def perform_graph_learning_tests(G, node_measurement_prob=node_measurement_prob, n_experiments=n_bio_experiments,
                                 relax_experiments=relax_experiments, timeout_seconds=timeout_seconds,
                                 max_attractor_length=max_attractor_length, tests_per_graph=tests_per_graph,
                                 limit_experiments_to_short_attractors=limit_experiments_to_short_attractors):
    """
    Given a model G, performs tests_per_graph tests, in each a random node's function is hidden, and a set
    of states in attractors are supplied to an ILP trying to find the function maximizing agreement with
    the states.
    :return: the average similarity between the learned function and hidden one in tests, the average proportion of
    experiment agreement in tests, and the average time taken per test.
    """
    # TODO: differentiate between tests in which the correct model was the correct solution vs otherwise.
    # TODO: better documentation here...

    test_times = []
    ilp_times = []
    solve_times = []
    test_agreements = []
    test_model_similarities = []
    
    # prepare attractor states to sample from
    graph_attractors = stochastic.estimate_attractors(G, n_walks=attractor_estimation_n_iter, with_basins=False)
    # graph_attractors = attractors.find_attractors_dubrova(G, attractors.dubrova_path, mutate_input_nodes=True)
    if limit_experiments_to_short_attractors:
        graph_attractors = [att for att in graph_attractors if len(att) <= max_attractor_length]
        if len(graph_attractors) == 0:
            print "warning! No attractors for of suitable length found in graph. Experiments cannot reconstruct"
            return 0, 0, 0
    attractor_states = list(itertools.chain.from_iterable(graph_attractors))

    for _ in range(tests_per_graph):
        test_start = time.time()  # TODO: measure ILP and experiment preparation time separately

        # hide a node's function
        G_hidden = G.copy()
        hidden_node_index = random.choice([i for i in range(len(G.vertices)) if
                                           len(G.vertices[i].predecessors()) > 0])
        G_hidden.vertices[hidden_node_index].function = None

        # create experiment measurements
        experiments = []
        for _ in range(n_experiments):
            experiment_state = random.choice(attractor_states)
            experiment = dict()
            for i in range(len(G.vertices)):
                # always measure input nodes
                if (len(G.vertices[i].predecessors()) == 0) or (random.random() < node_measurement_prob):
                    experiment[i] = experiment_state[i]
            assert len(experiment) != 0
            if len(experiment) != 0:
                experiments.append(experiment)
            else:
                print "Warning: Experiment came out with no measured nodes"


        # try to learn the model back
        ilp_start = time.time()
        G_found, agreement, solve_time = attractors.learn_model_from_experiment_agreement(
            G_hidden, experiments, relax_experiments,
            max_attractor_length, timeout_seconds, allow_suboptimal)
        solve_times.append(solve_time)
        ilp_times.append(time.time() - ilp_start)
        model_similarity = 0
        for var_combination in itertools.product((False, True),
                                                 repeat=len(G.vertices[hidden_node_index].predecessors())):
            if G.vertices[hidden_node_index].function(*var_combination) == \
                    G_found.vertices[hidden_node_index].function(*var_combination):
                model_similarity += 1<commit_msg>small bug fix in experiment model learning<commit_after>import cProfile
import random
import time
import graphs
import attractors
from shutil import copyfile
# from pebble import ProcessPool
from collections import namedtuple
import csv
import stochastic
import multiprocessing
import itertools
import numpy
import os
import sys
import signal, os, errno
from functools import wraps
import stat
import platform

attractor_estimation_n_iter = 100
node_measurement_prob = 0.8
n_bio_experiments = 50
relax_experiments = False
timeout_seconds = int(0.8 * 60 * 60)
graph_parent_dir = "cellcollective_models"
max_attractor_length = 5
graph_size_filter = 50
tests_per_graph = 20
limit_experiments_to_short_attractors = False
allow_suboptimal = True


def perform_graph_learning_tests(G, node_measurement_prob=node_measurement_prob, n_experiments=n_bio_experiments,
                                 relax_experiments=relax_experiments, timeout_seconds=timeout_seconds,
                                 max_attractor_length=max_attractor_length, tests_per_graph=tests_per_graph,
                                 limit_experiments_to_short_attractors=limit_experiments_to_short_attractors):
    """
    Given a model G, performs tests_per_graph tests, in each a random node's function is hidden, and a set
    of states in attractors are supplied to an ILP trying to find the function maximizing agreement with
    the states.
    :return: the average similarity between the learned function and hidden one in tests, the average proportion of
    experiment agreement in tests, and the average time taken per test.
    """
    # TODO: differentiate between tests in which the correct model was the correct solution vs otherwise.
    # TODO: better documentation here...

    test_times = []
    ilp_times = []
    solve_times = []
    test_agreements = []
    test_model_similarities = []
    
    # prepare attractor states to sample from
    graph_attractors = stochastic.estimate_attractors(G, n_walks=attractor_estimation_n_iter, with_basins=False)
    # graph_attractors = attractors.find_attractors_dubrova(G, attractors.dubrova_path, mutate_input_nodes=True)
    if limit_experiments_to_short_attractors:
        graph_attractors = [att for att in graph_attractors if len(att) <= max_attractor_length]
        if len(graph_attractors) == 0:
            print "warning! No attractors for of suitable length found in graph. Experiments cannot reconstruct"
            return 0, 0, 0
    attractor_states = list(itertools.chain.from_iterable(graph_attractors))

    for _ in range(tests_per_graph):
        test_start = time.time()  # TODO: measure ILP and experiment preparation time separately

        # hide a node's function
        G_hidden = G.copy()
        hidden_node_index = random.choice([i for i in range(len(G.vertices)) if
                                           len(G.vertices[i].predecessors()) > 0])
        G_hidden.vertices[hidden_node_index].function = None

        # create experiment measurements
        experiments = []
        for _ in range(n_experiments):
            experiment_state = random.choice(attractor_states)
            experiment = dict()
            for i in range(len(G.vertices)):
                # always measure input nodes
                if (len(G.vertices[i].predecessors()) == 0) or (random.random() < node_measurement_prob):
                    experiment[i] = experiment_state[i]
            assert len(experiment) != 0
            if len(experiment) != 0:
                experiments.append(experiment)
            else:
                print "Warning: Experiment came out with no measured nodes"


        # try to learn the model back
        ilp_start = time.time()
        G_found, agreement, solve_time = attractors.learn_model_from_experiment_agreement(
            G_hidden, experiments, relax_experiments,
            max_attractor_length, timeout_seconds, allow_suboptimal=allow_suboptimal)
        solve_times.append(solve_time)
        ilp_times.append(time.time() - ilp_start)
        model_similarity = 0
        for var_combination in itertools.product((False, True),
                                                 repeat=len(G.vertices[hidden_node_index].predecessors())):
            if G.vertices[hidden_node_index].function(*var_combination) == \
                    G_found.vertices[hidden_node_index].function(*var_combination):
                model_similarity += 1
{'fraction_non_alphanumeric': 0.06581332125056638, 'fraction_numerical': 0.006343452650657001, 'mean_word_length': 2.982408660351827, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1070079', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2401, 'n_words': 659}