<commit_before>import numpy as np
import random
import pubchem as pc
import chembl as ch
import utility

from math import floor, log, isnan, sqrt

# Number of label columns to prepare
INPUT_COUNT = 3
LABEL_COUNT = 1
USE_TEST_FLAGS = True

# Fixed alphasize options
ALPHA_FIXED = False
ALPHA_FIXED_SIZE = 42

# Epsilon for catching numbers close to zero
EPS = 0.0001

# Transforms data into 1 of k encoding
# Output format is 3D array of integers, representing positions of binary 1
def formatSMILES(rawData, col):
    print('  Formatting SMILES data column...')

    # Get a set of all used characters
    alphabet = set()
    for item in rawData:
        for char in item[col]:
            alphabet.add(char)

    # Map columns to letters
    colMapping = {}
    size = 1
    for char in alphabet:
        colMapping[char] = size
        size += 1

    maxLen = 0
    for item in rawData:
        maxLen = max(maxLen, len(item[col]))

    # DEBUG, data properties
    print("    Number of samples: {}".format(len(rawData)))
    print("    Maximum length of sample: {}".format(maxLen))
    print("    Size of alphabet: {}".format(size))

    if ALPHA_FIXED:
        size = ALPHA_FIXED_SIZE
    output = np.zeros((len(rawData), maxLen, size))

    itemCtr = 0
    for item in rawData:
        charCtr = 0
        for char in item[col]:
            charIdx = colMapping[char]
            output[itemCtr][charCtr][charIdx] = 1
            charCtr += 1
        for i in range(charCtr, maxLen):
            output[itemCtr][i][0] = 1
        itemCtr += 1

    print('  ...done')
    return size, maxLen, output

def formatNominal(rawData, timesteps, col):
    print('  Formatting nominal data column...')

    # Get a set of all possible values
    nominals = set()
    for item in rawData:
        nominals.add(item[col])

    # Map columns to nominals
    colMapping = {}
    size = 0
    for value in nominals:
        colMapping[value] = size
        size += 1

    print("    Number of samples: {}".format(len(rawData)))
    print("    Number of unique values: {}".format(size))

    itemCtr = 0
    output = np.zeros((len(rawData), timesteps, size))
    for item in rawData:
        valIdx = colMapping[item[col]]
        for step in range(timesteps):
            output[itemCtr][step][valIdx] = 1
        itemCtr += 1

    print('  ...done')
    return size, output


def formatSMILESEmbedded(rawData, col):
    print('  Formatting SMILES data column...')

    # Get a set of all used characters
    alphabet = set()
    for item in rawData:
        for char in item[col]:
            alphabet.add(char)

    # Map columns to letters
    colMapping = {}
    size = 1
    for char in alphabet:
        colMapping[char] = size
        size += 1

    maxLen = 0
    for item in rawData:
        maxLen = max(maxLen, len(item[col]))

    # DEBUG, data properties
    print("    Number of samples: {}".format(len(rawData)))
    print("    Maximum length of sample: {}".format(maxLen))

    output = np.zeros((len(rawData), maxLen))

    itemCtr = 0
    for item in rawData:
        charCtr = 0
        for char in item[col]:
            charIdx = colMapping[char]
            output[itemCtr][charCtr] = charIdx
            charCtr += 1
        for i in range(charCtr, maxLen):
            output[itemCtr][i] = 0
        itemCtr += 1

    print('  ...done')
    return size, maxLen, output


def formatNominalEmbedded(rawData, timesteps, output, col, shift):
    print('  Formatting nominal data column...')

    # Get a set of all possible values
    nominals = set()
    for item in rawData:
        nominals.add(item[col])

    # Map columns to nominals
    colMapping = {}
    size = 1
    for value in nominals:
        colMapping[value] = size
        size += 1<commit_msg>More documentation<commit_after>import numpy as np
import random
import pubchem as pc
import chembl as ch
import utility

from math import floor, log, isnan, sqrt

# Number of label columns to prepare
INPUT_COUNT = 3
LABEL_COUNT = 1
USE_TEST_FLAGS = True

# Fixed alphasize options
ALPHA_FIXED = False
ALPHA_FIXED_SIZE = 42

# Epsilon for catching numbers close to zero
EPS = 0.0001

# Transforms data into 1 of k encoding
# Output format is 3D array of integers, representing positions of binary 1
def formatSMILES(rawData, col):
    print('  Formatting SMILES data column...')

    # Get a set of all used characters
    alphabet = set()
    for item in rawData:
        for char in item[col]:
            alphabet.add(char)

    # Map columns to letters
    colMapping = {}
    size = 1
    for char in alphabet:
        colMapping[char] = size
        size += 1

    maxLen = 0
    for item in rawData:
        maxLen = max(maxLen, len(item[col]))

    # DEBUG, data properties
    print("    Number of samples: {}".format(len(rawData)))
    print("    Maximum length of sample: {}".format(maxLen))
    print("    Size of alphabet: {}".format(size))

    if ALPHA_FIXED:
        size = ALPHA_FIXED_SIZE
    output = np.zeros((len(rawData), maxLen, size))

    itemCtr = 0
    for item in rawData:
        charCtr = 0
        for char in item[col]:
            charIdx = colMapping[char]
            output[itemCtr][charCtr][charIdx] = 1
            charCtr += 1
        for i in range(charCtr, maxLen):
            output[itemCtr][i][0] = 1
        itemCtr += 1

    print('  ...done')
    return size, maxLen, output

def formatNominal(rawData, timesteps, col):
    print('  Formatting nominal data column...')

    # Get a set of all possible values
    nominals = set()
    for item in rawData:
        nominals.add(item[col])

    # Map columns to nominals
    colMapping = {}
    size = 0
    for value in nominals:
        colMapping[value] = size
        size += 1

    print("    Number of samples: {}".format(len(rawData)))
    print("    Number of unique values: {}".format(size))

    itemCtr = 0
    output = np.zeros((len(rawData), timesteps, size))
    for item in rawData:
        valIdx = colMapping[item[col]]
        for step in range(timesteps):
            output[itemCtr][step][valIdx] = 1
        itemCtr += 1

    print('  ...done')
    return size, output


# Use 2D format, containing indexes of one hot bit, suitable for usage in
# Embedding layers
def formatSMILESEmbedded(rawData, col):
    print('  Formatting SMILES data column...')

    # Get a set of all used characters
    alphabet = set()
    for item in rawData:
        for char in item[col]:
            alphabet.add(char)

    # Map columns to letters
    colMapping = {}
    size = 1
    for char in alphabet:
        colMapping[char] = size
        size += 1

    maxLen = 0
    for item in rawData:
        maxLen = max(maxLen, len(item[col]))

    # DEBUG, data properties
    print("    Number of samples: {}".format(len(rawData)))
    print("    Maximum length of sample: {}".format(maxLen))

    output = np.zeros((len(rawData), maxLen))

    itemCtr = 0
    for item in rawData:
        charCtr = 0
        for char in item[col]:
            charIdx = colMapping[char]
            output[itemCtr][charCtr] = charIdx
            charCtr += 1
        for i in range(charCtr, maxLen):
            output[itemCtr][i] = 0
        itemCtr += 1

    print('  ...done')
    return size, maxLen, output


# Same as formatSmilesEmbedded, but uses words instead of characters
# Shift parameter us used to encode multiple columns in one value for Embedding
def formatNominalEmbedded(rawData, timesteps, output, col, shift = 0):
    print('  Formatting nominal data column...')

    # Get a set of all possible values
    nominals = set()
    for item in rawData:
        nominals.add(item[col])

    # Map columns to nominals
    colMapping = {}
    size = 1
    for value in nominals:
        colMapping[value] = size
        size += 1
{'fraction_non_alphanumeric': 0.10026926529042185, 'fraction_numerical': 0.009744839081933581, 'mean_word_length': 2.346203346203346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541625', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2374, 'n_words': 826}