<commit_before>#!/usr/bin/python
from random import randint
from threading import Thread
from multiprocessing import Process, Queue, Manager
import timeit
from sys import exit
"""
Classes and methods for genetic algorithms.
To use this tool, the targe data stucture class should contains 3 methods which are RandomIndividual(), crossover() and mutation().
The fitness score must be numeric.
"""
# wait for all threads in a thread list
def join_all_threads(threads):
    for t in threads:
        t.join()

# wait for all processes in a process list
def join_all_processes(processes):
    for p in processes:
        p.join()

# change queue into a list
def Queue_to_list(queue):
    queue.put(None)
    l = []
    for item in iter(queue.get, None):
        l.append(item)
    return l
        
# Population class that is composed to CVGenetic class
class Population(object):
    def __init__(self, size):
        self.size = size
        self.individuals = []
        self.sorted = False
    
    def add(self, newindividual):
        if len(self.individuals) < self.size:
            self.individuals.append(newindividual)
            self.sorted = False
        else:
            least = self.get_least_index()
            if self.individuals[self.size-1][1] < newindividual[1]:
                self.individuals[self.size-1] = newindividual
            self.sorted = False
    
    # Get the index of individual with least fitness
    def get_least_index(self):
        if not self.sorted:
            self.sort()
        return len(self.individuals) - 1
    
    # Get N best fitness individual from population
    def get_best(self, N):
        if not self.sorted:
            self.sort()
        return self.individuals[:N]
    
    # sort individuals according to thier fitness
    def sort(self):
        if not self.sorted:
            self.individuals.sort(key = lambda t: t[1], reverse = True)
            self.sorted = True
            
    # Check existance of individual (Not used)
    def existed(self, i):
        for individual in self.individuals:
            if individual[0] == i:
                return True
        return False

class CVGenetic(object):
    def __init__(self, population_size, DataList, CalculateFitness, accuracy = 5, MutationRate = 0.2, output = True):
        print "Initializing Genetic Calculator"
        start = timeit.default_timer()
        self.population = Population(population_size)
        self.DataList = DataList
        self.accuracy = accuracy
        self.best = float("-inf")
        self.CalculateFitness = CalculateFitness
        self.output = output
        manager = Manager()
        self.store = manager.dict()
        newindividuals = Queue()
        processes = []
        for i in range(population_size):
            p = Process(target = self.create_newindividual, args = (DataList.RandomIndividual(), newindividuals))
            p.start()
            processes.append(p)
        join_all_processes(processes)
        newindividuals = Queue_to_list(newindividuals)
        for individual in newindividuals:
            self.population.add(individual)
        self.timer = 0
        self.MutationRate = MutationRate
        stop = timeit.default_timer()
        if self.output:
            print str(stop - start)+"s"
            print self.store
        
    def select(self, N):
        bests = self.population.get_best(N)
        if self.best == bests[0][0]:
            self.timer += 1
        else:
            self.best = bests[0][0]
            self.timer = 0
        if self.output:<commit_msg>added some comments<commit_after>#!/usr/bin/python
from random import randint
from threading import Thread
from multiprocessing import Process, Queue, Manager
import timeit
from sys import exit
"""
Classes and methods for genetic algorithms.
To use this tool, the targe data stucture class should contains 3 methods which are RandomIndividual(), crossover() and mutation().
The fitness score must be numeric.
If individual is a class object, the class need to have two additional methods (__hash__ and __eq__). 
"""
# wait for all threads in a thread list
def join_all_threads(threads):
    for t in threads:
        t.join()

# wait for all processes in a process list
def join_all_processes(processes):
    for p in processes:
        p.join()

# change queue into a list
def Queue_to_list(queue):
    # put 'None' at the end of the queue
    queue.put(None)
    l = []
    # iterator stops when meet 'None'
    for item in iter(queue.get, None):
        l.append(item)
    return l
        
# Population class that is composed to CVGenetic class
class Population(object):
    def __init__(self, size):
        self.size = size
        self.individuals = []
        self.sorted = False
    # add new individual to population
    def add(self, newindividual):
        if len(self.individuals) < self.size:
            self.individuals.append(newindividual)
            self.sorted = False
        else:
            least = self.get_least_index()
            if self.individuals[self.size-1][1] < newindividual[1]:
                self.individuals[self.size-1] = newindividual
            self.sorted = False
    
    # Get the index of individual with least fitness
    def get_least_index(self):
        if not self.sorted:
            self.sort()
        return len(self.individuals) - 1
    
    # Get N best fitness individual from population
    def get_best(self, N):
        if not self.sorted:
            self.sort()
        return self.individuals[:N]
    
    # sort individuals according to thier fitness
    def sort(self):
        if not self.sorted:
            self.individuals.sort(key = lambda t: t[1], reverse = True)
            self.sorted = True
            
    # Check existance of individual (Not used)
    def existed(self, i):
        for individual in self.individuals:
            if individual[0] == i:
                return True
        return False

class CVGenetic(object):
    def __init__(self, population_size, DataList, CalculateFitness, accuracy = 5, MutationRate = 0.2, output = True):
        print "Initializing Genetic Calculator"
        start = timeit.default_timer()
        self.population = Population(population_size)
        self.DataList = DataList
        self.accuracy = accuracy
        self.best = float("-inf")
        self.CalculateFitness = CalculateFitness
        self.output = output
        manager = Manager()
        self.store = manager.dict()
        newindividuals = Queue()
        processes = []
        # initilize population with random individual
        for i in range(population_size):
            p = Process(target = self.create_newindividual, args = (DataList.RandomIndividual(), newindividuals))
            p.start()
            processes.append(p)
        join_all_processes(processes)
        newindividuals = Queue_to_list(newindividuals)
        for individual in newindividuals:
            self.population.add(individual)
        self.timer = 0
        self.MutationRate = MutationRate
        stop = timeit.default_timer()
        if self.output:
            print str(stop - start)+"s"
            print self.store
        
    def select(self, N):
        bests = self.population.get_best(N)
        if self.best == bests[0][0]:
            self.timer += 1
        else:
            self.best = bests[0][0]
            self.timer = 0
        if self.output:
{'fraction_non_alphanumeric': 0.0854908306364617, 'fraction_numerical': 0.0048543689320388345, 'mean_word_length': 2.2275892080069624, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6029036', 'n_tokens_mistral': 2141, 'n_tokens_neox': 2028, 'n_words': 633}