<commit_before>from os.path import join
from copy import deepcopy

import lib.processors as processors
from lib.selectors import FileSelector

from config import DATA_PATH

class Factory:
    def __init__(self):
        self.root = DATA_PATH

    def Difference(self, dest):
        return Difference(join(self.root, dest))

    def LinearRegression(self, dest):
        return LinearRegression(dest)

    def MovingAverage(self, step, dest):
        return MovingAverage(step, join(self.root, dest))
        
    def WorkingGroup(self, cow, dest, drawer):
        return WorkingGroup(cow, join(self.root, dest), drawer)

class Manager:
    def __init__(self, dest):
        self.data = None
        self.dest = dest
        
        self.file_selector = FileSelector()
        
    def __rshift__(self, next):
        next.data = deepcopy(self.data)
        next.work()
        
    def work(self):
        data = self.file_selector.load(self.dest)
        
        if data is not None:
            self.data = data
        else:
            for series in self.data.series:
                series.x(), series.y() = self.processor.work(series.x(), series.y())
                
                self.data.drawer.add(series.x(), series.y(), series.name)
            
            self.data.drawer.save(self.dest)
            self.file_selector.save(self.data, self.dest)

class Difference(Manager):
    def __init__(self, dest):
        Manager.__init__(self, dest)
        
        self.processor = processors.Difference()

class LinearRegression(Manager):
    def __init__(self, dest):
        Manager.__init__(self, dest)
        
        self.processor = processors.LinearRegression()
        
    def work(self):
        pass

class MovingAverage(Manager):
    def __init__(self, step, dest):
        Manager.__init__(self, dest)
        
        self.step = max(1, int(step)) # 'step' is an integer greater than 1
        
        self.processor = processors.MovingAverage(self.step)

class Series:
    def __init__(self):
        self.name = []
        self.data = []
        
    def x(self):
        return self.data[0]
        
    def y(self):
        return self.data[1]
        
class WorkingGroup(Manager):
    def __init__(self, cow, dest, drawer=None):
        Manager.__init__(self, dest)
        
        self.cow = cow
        self.series = []
        
        self.data = self # To make __rshift__ work
        self.drawer = drawer
        self.processor = processors.Identity()
        
    def fill(self, name_getter, data_getters):
        names = name_getter["fn"](self.cow)
        
        for name in names:
            series = Series()
            series.name = name_getter["prefix"] + str(name) + name_getter["suffix"]
            
            for getter in data_getters:
                series.data.append(getter(self.cow, name))
            <commit_msg>Use dicts to store data<commit_after>from os.path import join

import lib.processors as processors
from lib.models import FileManager

from config import DATA_PATH

class Factory:
    def __init__(self):
        self.root = DATA_PATH

    def Difference(self, dest, force=False):
        return Difference(join(self.root, dest), force)

    def Identity(self, dest, force=False):
        return Identity(join(self.root, dest), force)

    def LinearRegression(self, percentage, dest, force=False):
        return LinearRegression(percentage, join(self.root, dest), force)

    def MovingAverage(self, step, dest, force=False):
        return MovingAverage(step, join(self.root, dest), force)
        
    def WorkingGroup(self, cow, drawer=None):
        return WorkingGroup(cow, drawer)

#
# Abstract managers
#
class Manager:
    def __init__(self, dest, force):
        self.data = None
        self.dest = dest
        self.force = force
        
        self.file_manager = FileManager()
        
    def work(self):
        raise RuntimeError("'work' method must be implemented.")
        
class XYManager(Manager):
    def __init__(self, dest, force):
        Manager.__init__(self, dest, force)
        
    def work(self, wg):
        if not self.force: 
            cache = self.file_manager.load(self.dest)
            
            if cache is not None:
                return cache
        
        # Build data
        for series in wg.series:
            x = series.data["x"]
            y = series.data["y"]
            
            x, y = self.processor.work(x, y)
            
            wg.drawer.add(x, y, series.name)
        
        wg.drawer.save(self.dest)
        self.file_manager.save(wg, self.dest)
        
        return wg

#
# Managers
#
class Difference(XYManager):
    def __init__(self, dest, force):
        XYManager.__init__(self, dest, force)
        
        self.processor = processors.Difference()

class Identity(XYManager):
    def __init__(self, dest, force):
        XYManager.__init__(self, dest, force)
        
        self.processor = processors.Identity()

class LinearRegression(Manager):
    def __init__(self, percentage, dest, force):
        Manager.__init__(self, dest, force)
        
        self.alea = processors.AleaValues(percentage)
        self.processor = processors.LinearRegression()
    
    def build_A(self, data):
        A = []
        l = len(data[0])
        
        for i in range(l):
            line = [1] # Offset
            
            for series in data:
                line.append(series[i])
                line.append(series[i]**2)
            
            A.append(line)
            
        return A
        
    def work(self, wg):
        if not self.force: 
            cache = self.file_manager.load(self.dest)
            
            if cache is not None:
                return cache
        
        # Build data
        for series in wg.series:
            cons = series.data["cons"]
            lact_days = series.data["lact_days"]
            lact = [series.id for i in range(len(cons))]
            
            B = series.data["prods"]
            A = self.build_A([cons, lact_days, lact])
            
            A_alea, B_alea = self.alea.work([A, B])
            
            series.data["A"] = A
            series.data["B_alea"] = B_alea
            series.data["A_alea"] = A_alea
            
            X = self.processor.work(A_alea, B_alea)
            series.data["X"] = X
            
            diff = self.processor.compare(A, X, B)
            series.data["diff"] = diff
            
            print(diff)
            
        # self.file_manager.save(wg, self.dest)
        
        return wg

class MovingAverage(XYManager):
    def __init__(self, step, dest, force):
        XYManager.__init__(self, dest, force)
        
        self.step = max(1, int(step)) # 'step' is an integer greater than 1
        
        self.processor = processors.MovingAverage(self.step)

#
# Working group
#
class Series:
    def __init__(self):
        self.id = -1
        self.name = ""
        self.data = {}
        
class WorkingGroup:
    def __init__(self, cow, drawer):
        self.cow = cow
        self.series = []
        
        self.drawer = drawer
        
    def fill(self, namer, getters):
        names = namer["getter"](self.cow)
        
        for name in names:
            series = Series()
            series.id = name
            series.name = namer["prefix"] + str(name) + namer["suffix"]
            
            for key, getter in getters.items():
                series.data[key] = getter(self.cow, name)
            
{'fraction_non_alphanumeric': 0.10440435128681348, 'fraction_numerical': 0.0013266118333775537, 'mean_word_length': 1.7544757033248082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136961', 'n_tokens_mistral': 2222, 'n_tokens_neox': 2160, 'n_words': 527}