<commit_before>import climate
import logging
import numpy as np
import os
import pickle
import theanets


def compress(source, k, key):
    logging.info('%s: looking for %s', source, key)

    filenames = sorted(
        fn for fn in os.listdir(source)
        if fn.endswith('{}.npy'.format(key)))
    arrays = [
        np.clip(np.load(os.path.join(source, fn), mmap_mode='r'), -1, 1)
        for fn in filenames]
    B, N = 128, arrays[0].shape[1]

    logging.info('loaded %s rows of %d-D data from %d files',
                 sum(len(a) for a in arrays), N, len(arrays))

    def batch():
        batch = np.zeros((B, N), 'f')
        for b in range(B):
            a = np.random.randint(len(arrays))
            j = np.random.randint(len(arrays[a]))
            batch[b] = arrays[a][j]
        return [batch]

    pca = theanets.Autoencoder([N, (k, 'linear'), (N, 'tied')])
    pca.train(batch, momentum=0.9)

    for arr, fn in zip(arrays, filenames):
        out = os.path.join(source, fn.replace(key, '{}_{}'.format(key, k)))
        karr = pca.encode(arr.astype('f'))
        logging.info('%s: saving %s', out, karr.shape)
        np.save(out, karr)

    out = os.path.join(source, 'pca_{}_{}.pkl'.format(key, k))
    pickle.dump(pca, open(out, 'wb'))


@climate.annotate(
    root='load data files from subject directories in this path',
    k=('compress to this many dimensions', 'option', None, int),
)
def main(root, k=400):
    for subject in sorted(os.listdir(root)):
        compress(os.path.join(root, subject), k, 'fjac')
        compress(os.path.join(root, subject), k, 'ijac')


if __name__ == '__main__':
    climate.call(main)
<commit_msg>Change compression to load data from CSVs.<commit_after>import climate
import glob
import gzip
import io
import lmj.cubes
import logging
import numpy as np
import os
import pandas as pd
import pickle
import theanets


def compress(source, k, activation, **kwargs):
    fns = sorted(glob.glob(os.path.join(source, '*', '*_jac.csv.gz')))
    logging.info('%s: found %d jacobians', source, len(fns))

    # the clipping operation affects about 2% of jacobian values.
    dfs = [np.clip(pd.read_csv(fn, index_col='time').dropna(), -10, 10)
           for fn in fns]

    B, N = 128, dfs[0].shape[1]

    logging.info('loaded %s rows of %d-D data from %d files',
                 sum(len(df) for df in dfs), N, len(dfs))

    def batch():
        batch = np.zeros((B, N), 'f')
        for b in range(B):
            a = np.random.randint(len(dfs))
            batch[b] = dfs[a].iloc[np.random.randint(len(dfs[a])), :]
        return [batch]

    pca = theanets.Autoencoder([N, (k, activation), (N, 'tied')])
    pca.train(batch, **kwargs)

    key = '{}_k{}'.format(activation, k)
    if 'hidden_l1' in kwargs:
        key += '_s' + kwargs['hidden_l1']

    for df, fn in zip(dfs, fns):
        df = pd.DataFrame(pca.encode(df), index=df.index)
        s = io.StringIO()
        df.to_csv(s, index_label='time')
        out = fn.replace('_jac', '_jac_' + key)
        with gzip.open(out, 'wb') as handle:
            handle.write(s.getvalue().encode('utf-8'))
        logging.info('%s: saved %s', out, df.shape)

    out = os.path.join(source, 'pca_{}.pkl'.format(key))
    pickle.dump(pca, open(out, 'wb'))


@climate.annotate(
    root='load data files from subject directories in this path',
    k=('compress to this many dimensions', 'option', None, int),
    activation=('use this activation function', 'option'),
)
def main(root, k=1000, activation='relu'):
    for subject in lmj.cubes.Experiment(root).subjects:
        compress(subject.root, k, activation,
                 momentum=0.9,
                 hidden_l1=0.9,
                 monitors={'hid1:out': (0.1, 0.5, 0.9)})


if __name__ == '__main__':
    climate.call(main)

{'fraction_non_alphanumeric': 0.16574440052700923, 'fraction_numerical': 0.01080368906455863, 'mean_word_length': 2.788423153692615, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5374558', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1339, 'n_words': 349}