<commit_before>
"""
Model process
-------------
This module contains the main class to execute the process.

TODO
----
matrix and corr in the same process
problem with reindices and matrix

"""

from os.path import join
import shelve
import networkx as nx
import numpy as np
import multiprocessing as mp
import time

from model_utils import filter_with_random_nets
from Mscthesis.IO.model_report import create_model_report
from aux_functions import init_compl_arrays


###############################################################################
################################## MAIN CLASS #################################
###############################################################################
class ModelProcess():
    """Abstract class for performs the process of computation of the models.
    ===============================
    Functionalities:
    - Compute net from data (parallel/sequential)
    - Compute net from precomputed neighs (parallel/sequential)
    - Compute net from agg points (parallel/sequential)
    - Compute matrix for trainning approach (parallel/sequential)

    ================================
    Problems:
    - Mask neighs
    - Get and compute descriptors (online/file)
    - Aggregate descriptors (measure dependant)

    ================================
    TODO:<commit_msg>Debug imports.<commit_after>
"""
Model process
-------------
This module contains the main class to execute the process.

TODO
----
matrix and corr in the same process
problem with reindices and matrix

"""

from os.path import join
import shelve
import networkx as nx
import numpy as np
import multiprocessing as mp
import time

from model_utils import filter_with_random_nets
from pySpatialTools.IO import create_model_report
from aux_functions import init_compl_arrays


###############################################################################
################################## MAIN CLASS #################################
###############################################################################
class ModelProcess():
    """Abstract class for performs the process of computation of the models.
    ===============================
    Functionalities:
    - Compute net from data (parallel/sequential)
    - Compute net from precomputed neighs (parallel/sequential)
    - Compute net from agg points (parallel/sequential)
    - Compute matrix for trainning approach (parallel/sequential)

    ================================
    Problems:
    - Mask neighs
    - Get and compute descriptors (online/file)
    - Aggregate descriptors (measure dependant)

    ================================
    TODO:
{'fraction_non_alphanumeric': 0.29035916824196595, 'fraction_numerical': 0.0, 'mean_word_length': 4.853982300884955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1067677', 'n_tokens_mistral': 632, 'n_tokens_neox': 573, 'n_words': 220}