<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
import datetime
import json
import random
import multiprocessing
import sys
import threading
import time
import imp
import re
import os
import sqlite3
import shutil

import numpy as np
from PIL import Image
import six
import cPickle as pickle
from six.moves import queue

import chainer
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers

VALIDATION_TIMING = 500 # ORIGINAL 50000

def load_image_list(path):
    tuples = []
    for line in open(path):
        pair = line.strip().split()
        tuples.append((pair[0], np.int32(pair[1])))
    return tuples

def read_image(path, mean_image, flip=False):
    image = np.asarray(Image.open(path))
    if image.ndim == 3:
        image = image.transpose(2, 0, 1)
    image = image.astype(np.float32)
    image -= mean_image
    image /= 255
    if flip and random.randint(0, 1) == 0:
        return np.fliplr(image)
    else:
        return image
 
def feed_data(train_list, val_list, mean_image, batchsize, val_batchsize, model, loaderjob, epoch, optimizer, data_q, avoid_flipping):
    i = 0
    count = 0
    x_batch = np.ndarray((batchsize, 3, model.insize, model.insize), dtype=np.float32)
    y_batch = np.ndarray((batchsize,), dtype=np.int32)
    val_x_batch = np.ndarray((val_batchsize, 3, model.insize, model.insize), dtype=np.float32)
    val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)
    
    batch_pool = [None] * batchsize
    val_batch_pool = [None] * val_batchsize
    pool = multiprocessing.Pool(loaderjob)
    data_q.put('train')
    use_flip = True
    
    if avoid_flipping == 1:
        use_flip = False
    
    for epoch in six.moves.range(1, 1 + epoch):
        print('epoch', epoch, file=sys.stderr)
        print('learning rate', optimizer.lr, file=sys.stderr)
        perm = np.random.permutation(len(train_list))
        for idx in perm:
            path, label = train_list[idx]
            batch_pool[i] = pool.apply_async(read_image, (path, mean_image, use_flip))
            y_batch[i] = label
            i += 1
            
            if i == batchsize:
                for j, x in enumerate(batch_pool):
                    x_batch[j] = x.get()
                data_q.put((x_batch.copy(), y_batch.copy(), epoch))
                i= 0
            
            count += 1
            if count % 1000 == 0:
                data_q.put('val')
                j = 0
                for path, label in val_list:
                    val_batch_pool[j] = pool.apply_async(read_image, (path, mean_image, False))
                    val_y_batch[j] = label
                    j += 1
                    
                    if j == val_batchsize:
                        for k, x in enumerate(val_batch_pool):
                            val_x_batch[k] = x.get()
                        data_q.put((val_x_batch.copy(), val_y_batch.copy(), epoch))
                        j = 0
                data_q.put('train')
                
        optimizer.lr *= 0.97
    pool.close()
    pool.join()
    data_q.put('end')
    return
    
def log_result(batchsize, val_batchsize, log_file,log_html, res_q):
    fH = open(log_html, 'w')
    fH.flush()

    f = open(log_file, 'w')
    f.write("count\tepoch\taccuracy\tloss\taccuracy(val)\tloss(val)\n")
    f.flush()
    count = 0<commit_msg> fixed image preparation logic <commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
import datetime
import json
import random
import multiprocessing
import sys
import threading
import time
import imp
import re
import os
import sqlite3
import shutil

import numpy as np
from PIL import Image
import six
import cPickle as pickle
from six.moves import queue

import chainer
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers

VALIDATION_TIMING = 500 # ORIGINAL 50000

def load_image_list(path):
    tuples = []
    for line in open(path):
        pair = line.strip().split()
        tuples.append((pair[0], np.int32(pair[1])))
    return tuples

def read_image(path, model_insize, mean_image, center=False, flip=False):
    cropwidth = 256 - model_insize
    image = np.asarray(Image.open(path)).transpose(2, 0, 1)
    if center:
        top = left = cropwidth / 2
    else:
        top = random.randint(0, cropwidth - 1)
        left = random.randint(0, cropwidth - 1)
    bottom = model_insize + top
    right = model_insize + left
    image = image[:, top:bottom, left:right].astype(np.float32)
    image -= mean_image[:, top:bottom, left:right]
    image /= 255
    if flip and random.randint(0, 1) == 0:
        return np.fliplr(image)
    else:
        return image
 
def feed_data(train_list, val_list, mean_image, batchsize, val_batchsize, model, loaderjob, epoch, optimizer, data_q, avoid_flipping):
    i = 0
    count = 0
    x_batch = np.ndarray((batchsize, 3, model.insize, model.insize), dtype=np.float32)
    y_batch = np.ndarray((batchsize,), dtype=np.int32)
    val_x_batch = np.ndarray((val_batchsize, 3, model.insize, model.insize), dtype=np.float32)
    val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)
    
    batch_pool = [None] * batchsize
    val_batch_pool = [None] * val_batchsize
    pool = multiprocessing.Pool(loaderjob)
    data_q.put('train')
    use_flip = True
    
    if avoid_flipping == 1:
        use_flip = False
    
    for epoch in six.moves.range(1, 1 + epoch):
        print('epoch', epoch, file=sys.stderr)
        print('learning rate', optimizer.lr, file=sys.stderr)
        perm = np.random.permutation(len(train_list))
        for idx in perm:
            path, label = train_list[idx]
            batch_pool[i] = pool.apply_async(read_image, (path, model.insize, mean_image, False, use_flip))
            y_batch[i] = label
            i += 1
            
            if i == batchsize:
                for j, x in enumerate(batch_pool):
                    x_batch[j] = x.get()
                data_q.put((x_batch.copy(), y_batch.copy(), epoch))
                i= 0
            
            count += 1
            if count % 1000 == 0:
                data_q.put('val')
                j = 0
                for path, label in val_list:
                    val_batch_pool[j] = pool.apply_async(read_image, (path, model.insize, mean_image, True, False))
                    val_y_batch[j] = label
                    j += 1
                    
                    if j == val_batchsize:
                        for k, x in enumerate(val_batch_pool):
                            val_x_batch[k] = x.get()
                        data_q.put((val_x_batch.copy(), val_y_batch.copy(), epoch))
                        j = 0
                data_q.put('train')
                
        optimizer.lr *= 0.97
    pool.close()
    pool.join()
    data_q.put('end')
    return
    
def log_result(batchsize, val_batchsize, log_file,log_html, res_q):
    fH = open(log_html, 'w')
    fH.flush()

    f = open(log_file, 'w')
    f.write("count\tepoch\taccuracy\tloss\taccuracy(val)\tloss(val)\n")
    f.flush()
    count = 0
{'fraction_non_alphanumeric': 0.10832985676540119, 'fraction_numerical': 0.016548463356973995, 'mean_word_length': 2.161318681318681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5839896', 'n_tokens_mistral': 2441, 'n_tokens_neox': 2312, 'n_words': 575}