<commit_before>import datajoint as dj
from stack import Stack
from bernoulli import FullBernoulliProcess, RankDegenerateBernoulliProcess
from utils import *
import seaborn as sns
from djaddon import gitlog

schema = dj.schema('datajoint_aod_cell_detection', locals())
import git
import itertools

preprocessors = {
    'center_medianfilter_unsharpmask_histeq':
        lambda x: histeq(unsharp_masking(medianfilter(center(x.squeeze()))), 500).mean(axis=-1),
    'center_medianfilter':
        lambda x: medianfilter(center(x.squeeze())).mean(axis=-1),
    'center_medianfilter_unsharpmask':
        lambda x: unsharp_masking(medianfilter(center(x.squeeze()))).mean(axis=-1),
    'center_medianfilter_unsharpmask_whiten':
        lambda x: whiten(unsharp_masking(medianfilter(center(x.squeeze())))).mean(axis=-1),
}


@schema
class Stacks(dj.Lookup):
    definition = """
    # input training files

    file_name   : varchar(100)  # filename
    vx          : int # x voxel size
    vy          : int # y voxel size
    vz          : int # z voxel size
    ---
    """

    contents = [
        ('data/2015-08-25_12-49-41_2015-08-25_13-02-18.h5', 17, 17, 15),
        ('data/2015-08-25_13-49-54_2015-08-25_13-57-23.h5', 17, 17, 15),
        ('data/2015-08-25_14-36-29_2015-08-25_14-44-41.h5', 17, 17, 15),
    ]


# @schema
# class TrainTestPairs(dj.Lookup):
#     definition = """
#     # input test files
#     ->Stacks
#     test_file_name   : varchar(100)  # filename
#     ---
#     """
#
#     def _prepare(self):
#         self.insert( (Stacks()*Stacks().project(test_file_name='file_name') - 'file_name = test_file_name').fetch(),
#                      skip_duplicates=True)
#

@schema
class ComponentNumbers(dj.Lookup):
    definition = """
    # number of linear and quadratic components for parameter selection

    linear_components          : int # number of linear components
    quadratic_components       : int # number of quadratic components
    common_components          : int
    ---

    """

    @property
    def contents(self):
        yield from itertools.starmap(lambda a, b: a + b,
                                itertools.product(zip(range(2, 12, 2), range(2, 12, 2)), zip(range(2, 12, 2))))

@schema
class Repetitions(dj.Lookup):
    definition = """
    # multiple repetitions for fitting

    idx     : int # repetition index
    ---

    """

    @property
    def contents(self):
        yield from zip(range(10))


@schema
class Preprocessing(dj.Lookup):
    definition = """
    preprocessing       : varchar(200)
    """

    @property
    def contents(self):
        return [(k,) for k in preprocessors]


@schema
@gitlog
class TrainedBSTM(dj.Computed):
    definition = """
    # trained BSTM models

    -> Stacks
    -> ComponentNumbers
    -> Repetitions
    -> Preprocessing
    ---

    u_xy                     : longblob # quadratic filters xy
    u_z                      : longblob # quadratic filters z
    w_xy                     : longblob # linear filters xy
    w_z                      : longblob # linear filters z
    beta                     : longblob # quadratic coefficients
    gamma                    : longblob # linear  coefficients
    b                        : longblob # offsets
    train_cross_entropy      : double   # in Bits/component
    train_auc                : double   # in Bits/component
    train_auc_weighted       : double   # in Bits/component
    """

    def _make_tuples(self, key):
        key_sub = dict(key)
        s = Stack(key['file_name'], preprocessor=preprocessors[key['preprocessing']])
        voxel = (Stacks() & key).fetch1['vx', 'vy', 'vz']
        b = RankDegenerateBernoulliProcess(voxel,
                                           quadratic_channels=key['quadratic_components'],
                                           linear_channels=key['linear_components'],
                                           common_channels=key['common_components']
                                           )
        b.fit(s.X, s.cells, maxiter=100)
        key.update(b.parameters)
        key['train_cross_entropy'] = b.cross_entropy(s.X, s.cells)
        key['train_auc_weighted'] = b.auc(s.X, s.cells, average='weighted')
        key['train_auc'] = b.auc(s.X, s.cells, average='macro')
        self.insert1(key)

    def key2BSTM(self, key):
        trained = (self() & key).fetch1()
        voxel = key['vx'], key['vy'], key['vz']
        b = RankDegenerateBernoulliProcess(voxel, quadratic_channels=key['quadratic_components'],
                                           linear_channels=key['linear_components'],
                                           common_channels=key['common_components'])
        b.set_parameters(**trained)
        return b


@gitlog
@schema
class TestedBSTM(dj.Computed):
    definition = """<commit_msg>fix gitlog key order<commit_after>import datajoint as dj
from stack import Stack
from bernoulli import FullBernoulliProcess, RankDegenerateBernoulliProcess
from utils import *
import seaborn as sns
from djaddon import gitlog

schema = dj.schema('datajoint_aod_cell_detection', locals())
import git
import itertools

preprocessors = {
    'center_medianfilter_unsharpmask_histeq':
        lambda x: histeq(unsharp_masking(medianfilter(center(x.squeeze()))), 500).mean(axis=-1),
    'center_medianfilter':
        lambda x: medianfilter(center(x.squeeze())).mean(axis=-1),
    'center_medianfilter_unsharpmask':
        lambda x: unsharp_masking(medianfilter(center(x.squeeze()))).mean(axis=-1),
    'center_medianfilter_unsharpmask_whiten':
        lambda x: whiten(unsharp_masking(medianfilter(center(x.squeeze())))).mean(axis=-1),
}


@schema
class Stacks(dj.Lookup):
    definition = """
    # input training files

    file_name   : varchar(100)  # filename
    vx          : int # x voxel size
    vy          : int # y voxel size
    vz          : int # z voxel size
    ---
    """

    contents = [
        ('data/2015-08-25_12-49-41_2015-08-25_13-02-18.h5', 17, 17, 15),
        ('data/2015-08-25_13-49-54_2015-08-25_13-57-23.h5', 17, 17, 15),
        ('data/2015-08-25_14-36-29_2015-08-25_14-44-41.h5', 17, 17, 15),
    ]


# @schema
# class TrainTestPairs(dj.Lookup):
#     definition = """
#     # input test files
#     ->Stacks
#     test_file_name   : varchar(100)  # filename
#     ---
#     """
#
#     def _prepare(self):
#         self.insert( (Stacks()*Stacks().project(test_file_name='file_name') - 'file_name = test_file_name').fetch(),
#                      skip_duplicates=True)
#

@schema
class ComponentNumbers(dj.Lookup):
    definition = """
    # number of linear and quadratic components for parameter selection

    linear_components          : int # number of linear components
    quadratic_components       : int # number of quadratic components
    common_components          : int
    ---

    """

    @property
    def contents(self):
        yield from itertools.starmap(lambda a, b: a + b,
                                itertools.product(zip(range(2, 12, 2), range(2, 12, 2)), zip(range(2, 12, 2))))

@schema
class Repetitions(dj.Lookup):
    definition = """
    # multiple repetitions for fitting

    idx     : int # repetition index
    ---

    """

    @property
    def contents(self):
        yield from zip(range(10))


@schema
class Preprocessing(dj.Lookup):
    definition = """
    preprocessing       : varchar(200)
    """

    @property
    def contents(self):
        return [(k,) for k in preprocessors]


@schema
@gitlog
class TrainedBSTM(dj.Computed):
    definition = """
    # trained BSTM models

    -> Stacks
    -> ComponentNumbers
    -> Repetitions
    -> Preprocessing
    ---

    u_xy                     : longblob # quadratic filters xy
    u_z                      : longblob # quadratic filters z
    w_xy                     : longblob # linear filters xy
    w_z                      : longblob # linear filters z
    beta                     : longblob # quadratic coefficients
    gamma                    : longblob # linear  coefficients
    b                        : longblob # offsets
    train_cross_entropy      : double   # in Bits/component
    train_auc                : double   # in Bits/component
    train_auc_weighted       : double   # in Bits/component
    """

    def _make_tuples(self, key):
        key_sub = dict(key)
        s = Stack(key['file_name'], preprocessor=preprocessors[key['preprocessing']])
        voxel = (Stacks() & key).fetch1['vx', 'vy', 'vz']
        b = RankDegenerateBernoulliProcess(voxel,
                                           quadratic_channels=key['quadratic_components'],
                                           linear_channels=key['linear_components'],
                                           common_channels=key['common_components']
                                           )
        b.fit(s.X, s.cells, maxiter=100)
        key.update(b.parameters)
        key['train_cross_entropy'] = b.cross_entropy(s.X, s.cells)
        key['train_auc_weighted'] = b.auc(s.X, s.cells, average='weighted')
        key['train_auc'] = b.auc(s.X, s.cells, average='macro')
        self.insert1(key)

    def key2BSTM(self, key):
        trained = (self & key).fetch1()
        voxel = key['vx'], key['vy'], key['vz']
        b = RankDegenerateBernoulliProcess(voxel, quadratic_channels=key['quadratic_components'],
                                           linear_channels=key['linear_components'],
                                           common_channels=key['common_components'])
        b.set_parameters(**trained)
        return b


@schema
@gitlog
class TestedBSTM(dj.Computed):
    definition = """
{'fraction_non_alphanumeric': 0.11838973403704854, 'fraction_numerical': 0.029390458449756806, 'mean_word_length': 2.127508090614887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1006668', 'n_tokens_mistral': 3202, 'n_tokens_neox': 2859, 'n_words': 595}