<commit_before>"""Creates coarse classifier training datasets from clip HDF5 files."""


from pathlib import Path
import time

import h5py
import numpy as np
import tensorflow as tf

import vesper.util.os_utils as os_utils
import vesper.signal.resampling_utils as resampling_utils


VALIDATION_FRACTION = .1
TEST_FRACTION = .1

CALL_TYPE = 'Tseep'

DATA_DIR_PATH = Path(
    '/Users/harold/Desktop/NFC/Data/Vesper ML/NFC Time Bounds Marker 1.0/')

INPUT_DIR_PATH = DATA_DIR_PATH / 'HDF5 Files' / CALL_TYPE

THRUSH_INPUT_FILE_NAMES = ''

TSEEP_INPUT_FILE_NAMES = '''
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Angela.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Bear.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Bell Crossing.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Darby.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Dashiell.h5
'''.strip().split('\n')

INPUT_FILE_NAMES = {
    'Thrush': THRUSH_INPUT_FILE_NAMES,
    'Tseep': TSEEP_INPUT_FILE_NAMES,
}[CALL_TYPE]

OUTPUT_SAMPLE_RATE = 24000

OUTPUT_DIR_PATH = DATA_DIR_PATH / 'Datasets' / CALL_TYPE
OUTPUT_FILE_NAME_FORMAT = '{}_{}.tfrecords'


def main():
    
    for input_file_name in INPUT_FILE_NAMES:
        
        # Get clip IDs from input file.
        input_file_path = INPUT_DIR_PATH / input_file_name
        file_ = h5py.File(input_file_path, 'r')
        clip_group = file_['clips']
        clip_ids = list(clip_group.keys())
        
        # Shuffle IDs.
        shuffled_clip_ids = np.random.permutation(clip_ids)
        
        # Partition shuffled IDs for training, validation, and test datasets.
        train_clip_ids, val_clip_ids, test_clip_ids = \
            partition_clip_ids(shuffled_clip_ids)
        
        # Create tfrecord files.
        file_name_prefix = input_file_path.stem
        create_tfrecord_file(
            file_name_prefix, 'Training', clip_group, train_clip_ids)
        create_tfrecord_file(
            file_name_prefix, 'Validation', clip_group, val_clip_ids)
        create_tfrecord_file(
            file_name_prefix, 'Test', clip_group, test_clip_ids)
        
        
def partition_clip_ids(ids):
    
    clip_count = len(ids)
    
    val_start_index = get_subset_start_index(
        clip_count, VALIDATION_FRACTION + TEST_FRACTION)
    
    test_start_index = get_subset_start_index(clip_count, TEST_FRACTION)
    
    train_ids = ids[:val_start_index]
    val_ids = ids[val_start_index:test_start_index]
    test_ids = ids[test_start_index:]
    
    return train_ids, val_ids, test_ids

    
def get_subset_start_index(clip_count, offset):
    return int(round(clip_count * (1 - offset)))


def create_tfrecord_file(file_name_prefix, dataset_name, clip_group, clip_ids):
    
    start_time = time.time()
    
    print('        Creating TF examples...')
    
    examples = create_tf_examples(clip_group, clip_ids)
    
    print('        Writing TF examples to file...')
    
    file_path = get_output_file_path(file_name_prefix, dataset_name)
    os_utils.create_parent_directory(file_path)
    with tf.io.TFRecordWriter(str(file_path)) as writer:
        for example in examples:
            writer.write(example.SerializeToString())
             
    end_time = time.time()
    delta = end_time - start_time
    rate = len(clip_ids) / delta
    print(
        f'    Wrote {len(clip_ids)} {dataset_name} clips in {delta:.1f} '
        f'seconds, a rate of {rate:.1f} clips per second.')


def create_tf_examples(clip_group, clip_ids):
    return [create_tf_example(clip_group, i) for i in clip_ids]
    

def create_tf_example(clip_group, clip_id):
    
    clip = clip_group[clip_id]
    
    samples, call_start_index, call_end_index = get_clip_data(clip)
    
    samples_feature = create_bytes_feature(samples.tobytes())
    clip_id_feature = create_int64_feature(int(clip_id))
    start_index_feature = create_int64_feature(call_start_index)
    end_index_feature = create_int64_feature(call_end_index)<commit_msg>Made minor changes to NFC time bound marker dataset creation script.<commit_after>"""Creates NFC time bound marker datasets from clip HDF5 files."""


from pathlib import Path
import time

import h5py
import numpy as np
import tensorflow as tf

import vesper.util.os_utils as os_utils
import vesper.signal.resampling_utils as resampling_utils


VALIDATION_FRACTION = .1
TEST_FRACTION = .1

CALL_TYPE = 'Tseep'

DATA_DIR_PATH = Path(
    '/Users/harold/Desktop/NFC/Data/Vesper ML/NFC Time Bound Marker 1.0/')

INPUT_DIR_PATH = DATA_DIR_PATH / 'HDF5 Files' / CALL_TYPE

THRUSH_INPUT_FILE_NAMES = ''

TSEEP_INPUT_FILE_NAMES = '''
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Angela.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Bear.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Bell Crossing.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Darby.h5
Tseep_2017 MPG Ranch 30k_Old Bird Redux 1.1_Dashiell.h5
'''.strip().split('\n')

INPUT_FILE_NAMES = {
    'Thrush': THRUSH_INPUT_FILE_NAMES,
    'Tseep': TSEEP_INPUT_FILE_NAMES,
}[CALL_TYPE]

OUTPUT_SAMPLE_RATE = 24000

OUTPUT_DIR_PATH = DATA_DIR_PATH / 'Datasets' / CALL_TYPE
OUTPUT_FILE_NAME_FORMAT = '{}_{}.tfrecords'


def main():
    
    for input_file_name in INPUT_FILE_NAMES:
        
        # Get clip IDs from input file.
        input_file_path = INPUT_DIR_PATH / input_file_name
        file_ = h5py.File(input_file_path, 'r')
        clip_group = file_['clips']
        clip_ids = list(clip_group.keys())
        
        # Shuffle IDs.
        shuffled_clip_ids = np.random.permutation(clip_ids)
        
        # Partition shuffled IDs for training, validation, and test datasets.
        train_clip_ids, val_clip_ids, test_clip_ids = \
            partition_clip_ids(shuffled_clip_ids)
        
        # Create tfrecord files.
        file_name_prefix = input_file_path.stem
        create_tfrecord_file(
            file_name_prefix, 'Training', clip_group, train_clip_ids)
        create_tfrecord_file(
            file_name_prefix, 'Validation', clip_group, val_clip_ids)
        create_tfrecord_file(
            file_name_prefix, 'Test', clip_group, test_clip_ids)
        
        
def partition_clip_ids(ids):
    
    clip_count = len(ids)
    
    val_start_index = get_subset_start_index(
        clip_count, VALIDATION_FRACTION + TEST_FRACTION)
    
    test_start_index = get_subset_start_index(clip_count, TEST_FRACTION)
    
    train_ids = ids[:val_start_index]
    val_ids = ids[val_start_index:test_start_index]
    test_ids = ids[test_start_index:]
    
    return train_ids, val_ids, test_ids

    
def get_subset_start_index(clip_count, offset):
    return int(round(clip_count * (1 - offset)))


def create_tfrecord_file(file_name_prefix, dataset_name, clip_group, clip_ids):
    
    start_time = time.time()
    
    file_path = get_output_file_path(file_name_prefix, dataset_name)

    print(f'Writing {len(clip_ids)} clips to file "{file_path}"...')
    examples = create_tf_examples(clip_group, clip_ids)
    
    os_utils.create_parent_directory(file_path)
    with tf.io.TFRecordWriter(str(file_path)) as writer:
        for example in examples:
            writer.write(example.SerializeToString())
             
    end_time = time.time()
    delta = end_time - start_time
    rate = len(clip_ids) / delta
    print(
        f'    Wrote {len(clip_ids)} {dataset_name} clips in {delta:.1f} '
        f'seconds, a rate of {rate:.1f} clips per second.')


def create_tf_examples(clip_group, clip_ids):
    return [create_tf_example(clip_group, i) for i in clip_ids]
    

def create_tf_example(clip_group, clip_id):
    
    clip = clip_group[clip_id]
    
    samples, call_start_index, call_end_index = get_clip_data(clip)
    
    samples_feature = create_bytes_feature(samples.tobytes())
    clip_id_feature = create_int64_feature(int(clip_id))
    start_index_feature = create_int64_feature(call_start_index)
    end_index_feature = create_int64_feature(call_end_index)
{'fraction_non_alphanumeric': 0.08202927814235235, 'fraction_numerical': 0.01691065118626956, 'mean_word_length': 3.4199665365309535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4665739', 'n_tokens_mistral': 2887, 'n_tokens_neox': 2761, 'n_words': 560}