<commit_before>#! /usr/bin/env python
#
# Copyright (C) 2016 <NAME> <<EMAIL>>
# License: 3-clause BSD

"""
# skchem.data.converters.chembl

Dataset constructor for ChEMBL
"""

import pandas as pd
import os

from .base import Converter, default_pipeline, contiguous_order, Feature
from ...cross_validation import SimThresholdSplit
from ... import descriptors


class ChEMBLConverter(Converter):

    """ Converter for the ChEMBL dataset. """

    def __init__(self, directory, output_directory, output_filename='chembl.h5'):

        output_path = os.path.join(output_directory, output_filename)

        infile = os.path.join(directory, 'chembl_raw.h5')
        ms, y = self.parse_infile(infile)

        pipeline = default_pipeline()

        ms, y = pipeline.transform_filter(ms, y)

        cv = SimThresholdSplit(min_threshold=0.6, n_jobs=-1).fit(ms)
        train, valid, test = cv.split((70, 15, 15))
        (ms, y, train, valid, test) = contiguous_order((ms, y, train, valid, test), (train, valid, test))
        splits = (('train', train), ('valid', valid), ('test', test))

        features = (
        Feature(fper=descriptors.MorganFeaturizer(),
                key='X_morg',
                axis_names=['batch', 'features']),
        Feature(fper=descriptors.PhysicochemicalFeaturizer(),
                key='X_pc',
                axis_names=['batch', 'features']),
        Feature(fper=descriptors.AtomFeaturizer(max_atoms=100),
                key='A',
                axis_names=['batch', 'atom_idx', 'features']),
        Feature(fper=descriptors.GraphDistanceTransformer(max_atoms=100),
                key='G',
                axis_names=['batch', 'atom_idx', 'atom_idx']),
        Feature(fper=descriptors.SpacialDistanceTransformer(max_atoms=100),
                key='G_d'))

        self.run(ms, y, output_path, splits=splits)


    def parse_infile(self, filename):

        ms = pd.read_hdf(filename, 'structure')
        y = pd.read_hdf(filename, 'targets/Y')
        return ms, y<commit_msg>added module level running for chembl dataset<commit_after>#! /usr/bin/env python
#
# Copyright (C) 2016 <NAME> <<EMAIL>>
# License: 3-clause BSD

"""
# skchem.data.converters.chembl

Dataset constructor for ChEMBL
"""
import logging
import pandas as pd
import os

from .base import Converter, default_pipeline, contiguous_order, Feature
from ...cross_validation import SimThresholdSplit
from ... import descriptors

LOGGER = logging.getLogger(__name__)


class ChEMBLConverter(Converter):

    """ Converter for the ChEMBL dataset. """

    def __init__(self, directory, output_directory, output_filename='chembl.h5'):

        output_path = os.path.join(output_directory, output_filename)

        infile = os.path.join(directory, 'chembl_raw.h5')
        ms, y = self.parse_infile(infile)

        pipeline = default_pipeline()

        ms, y = pipeline.transform_filter(ms, y)

        cv = SimThresholdSplit(min_threshold=0.6, n_jobs=-1).fit(ms)
        train, valid, test = cv.split((70, 15, 15))
        (ms, y, train, valid, test) = contiguous_order((ms, y, train, valid, test), (train, valid, test))
        splits = (('train', train), ('valid', valid), ('test', test))

        features = (
        Feature(fper=descriptors.MorganFeaturizer(),
                key='X_morg',
                axis_names=['batch', 'features']),
        Feature(fper=descriptors.PhysicochemicalFeaturizer(),
                key='X_pc',
                axis_names=['batch', 'features']),
        Feature(fper=descriptors.AtomFeaturizer(max_atoms=100),
                key='A',
                axis_names=['batch', 'atom_idx', 'features']),
        Feature(fper=descriptors.GraphDistanceTransformer(max_atoms=100),
                key='G',
                axis_names=['batch', 'atom_idx', 'atom_idx']),
        Feature(fper=descriptors.SpacialDistanceTransformer(max_atoms=100),
                key='G_d'))

        self.run(ms, y, output_path, splits=splits)


    def parse_infile(self, filename):

        ms = pd.read_hdf(filename, 'structure')
        y = pd.read_hdf(filename, 'targets/Y')
        return ms, y

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    LOGGER.info('Converting ChEMBL...')
    ChEMBLConverter.convert()
{'fraction_non_alphanumeric': 0.13716295427901523, 'fraction_numerical': 0.011723329425556858, 'mean_word_length': 3.0056338028169014, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1763093', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1323, 'n_words': 274}