<commit_before>import os
import operator
import numpy as np
import pandas as pd

from shutil import copyfile

import torch
import torch.nn as nn
from torch.autograd import Variable

from itertools import combinations

from gensim.matutils import unitvec

from pymongo.errors import CursorNotFound
from pymongo import MongoClient

from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

from tqdm import tqdm

try:
    import matplotlib.pyplot as plt
except ImportError as err:
    plt = None
    print(err)

from tempfile import gettempdir

import argparse

aa_sim = pd.read_csv('Data/aa_sim.csv')
aa_unlike = [np.where(aa_sim.loc[[w], :] == 0)[1] for w in range(0, 25)]
AA = aa_sim.columns
dictionary, reverse_dictionary = dict(zip(AA, range(25))), dict(zip(range(25), AA))
vocabulary_size = len(AA)
n_clstr = 8

print(dictionary)
assert vocabulary_size == 25


class BatchLoader(object):
    def __init__(self, win_size, batch_size, train):
        self.win_size = win_size
        self.batch_size = batch_size
        self.train = train
        self.batch_buffer = np.ndarray([])
        self.labels_buffer = np.ndarray([])

    def __iter__(self):

        if self.train:
            self.stream = collection.aggregate([{"$sample": {"size": args.size_train}}])
            # pbar = tqdm(range(args.size_train), "sequences loaded")
        else:
            self.stream = collection.aggregate([{"$sample": {"size": args.size_test}}])
            # pbar = tqdm(range(args.size_test), "sequences loaded")

        seq = self._get_sequence()
        # pbar.update(1)
        seq_pos = 0

        batch_buffer, labels_buffer = self.batch_buffer, self.labels_buffer

        while self.stream.alive:

            batch_buffer, labels_buffer, seq_pos, batch_pos = \
                self._get_batch(seq, batch_buffer, labels_buffer, batch_pos=0, seq_pos=seq_pos)

            if seq_pos == 0:  # seq finished
                try:
                    seq = self._get_sequence()
                    # pbar.update(1)
                except CursorNotFound:
                    print("Cursor not found")
                    break

                batch_buffer, labels_buffer, seq_pos, batch_pos = \
                    self._get_batch(seq, batch_buffer, labels_buffer, batch_pos=batch_pos, seq_pos=0)

            else:
                yield batch_buffer, labels_buffer

        # pbar.close()

    def _get_batch(self, seq, batch, labels, batch_pos=0, seq_pos=0):
        return batch, labels, seq_pos, batch_pos

    def _get_sequence(self):
        return ''
<commit_msg>copied batch buffers<commit_after>import os
import operator
import numpy as np
import pandas as pd

from shutil import copyfile

import torch
import torch.nn as nn
from torch.autograd import Variable

from itertools import combinations

from gensim.matutils import unitvec

from pymongo.errors import CursorNotFound
from pymongo import MongoClient

from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

from tqdm import tqdm

try:
    import matplotlib.pyplot as plt
except ImportError as err:
    plt = None
    print(err)

from tempfile import gettempdir

import argparse

aa_sim = pd.read_csv('Data/aa_sim.csv')
aa_unlike = [np.where(aa_sim.loc[[w], :] == 0)[1] for w in range(0, 25)]
AA = aa_sim.columns
dictionary, reverse_dictionary = dict(zip(AA, range(25))), dict(zip(range(25), AA))
vocabulary_size = len(AA)
n_clstr = 8

print(dictionary)
assert vocabulary_size == 25


class BatchLoader(object):
    def __init__(self, win_size, batch_size, train):
        self.win_size = win_size
        self.batch_size = batch_size
        self.train = train
        self.batch_buffer = np.ndarray([])
        self.labels_buffer = np.ndarray([])

    def __iter__(self):

        if self.train:
            self.stream = collection.aggregate([{"$sample": {"size": args.size_train}}])
            # pbar = tqdm(range(args.size_train), "sequences loaded")
        else:
            self.stream = collection.aggregate([{"$sample": {"size": args.size_test}}])
            # pbar = tqdm(range(args.size_test), "sequences loaded")

        seq = self._get_sequence()
        # pbar.update(1)
        seq_pos = 0

        batch_buffer, labels_buffer = self.batch_buffer, self.labels_buffer

        while self.stream.alive:

            batch_buffer, labels_buffer, seq_pos, batch_pos = \
                self._get_batch(seq, batch_buffer, labels_buffer, batch_pos=0, seq_pos=seq_pos)

            if seq_pos == 0:  # seq finished
                try:
                    seq = self._get_sequence()
                    # pbar.update(1)
                except CursorNotFound:
                    print("Cursor not found")
                    break

                batch_buffer, labels_buffer, seq_pos, batch_pos = \
                    self._get_batch(seq, batch_buffer, labels_buffer, batch_pos=batch_pos, seq_pos=0)

            else:
                yield np.copy(batch_buffer), np.copy(labels_buffer)

        # pbar.close()

    def _get_batch(self, seq, batch, labels, batch_pos=0, seq_pos=0):
        return batch, labels, seq_pos, batch_pos

    def _get_sequence(self):
        return ''

{'fraction_non_alphanumeric': 0.09659631108575775, 'fraction_numerical': 0.007606008746910059, 'mean_word_length': 2.6351071181755357, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3063357', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1594, 'n_words': 410}