<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import os, sys, re
import random
from time import time

import cv2
import numpy as np

import chainer
from chainer import cuda
import chainer.functions as F
import overwrite
from chainer.functions import caffe

def getNeuralCode(directory, layer="fc6", gpu = -1):

  model = "bvlc_alexnet.caffemodel"
  #use alexnet
  print('alexnet is being loaded!')
  #calculate load time
  timeMemory = time()
  func = caffe.CaffeFunction(model)
  print('alexnet was loaded!')
  print('It took ' + str(int(time() - timeMemory)) + " secondes")

  #gpu mode
  if gpu >= 0:
    cuda.init(gpu)
    func.to_gpu()

  in_size = 227
  mean_image = np.load("ilsvrc_2012_mean.npy")

  # return neural code
  print("neural code is extraced from layer " + layer)
  def neuralCode(x):
    y, = func(inputs={'data': x}, outputs=[layer], train=False)
    return y.data[0]

  cropwidth = 256 - in_size
  start = cropwidth // 2
  stop = start + in_size
  mean_image = mean_image[:, start:stop, start:stop].copy()
  target_shape = (256, 256)
  output_side_length=256

  numPic = 0
  #count pictures
  for folderPath in directory:
    #search pictures
    picturePath = [picture for picture in os.listdir(folderPath)
                   if re.findall(r"\.png$|\.jpg$|\.JPG$|\.PNG$|\.JPEG$",picture)]
    print("you have " + str(len(picturePath)) + " pictures in " + folderPath)
    numPic = numPic + len(picturePath)
  
  print("you have totally" + str(numPic) + " pictures")
  answer = {}
  count = 0
  for folderPath in directory:
    #search pictures
    picturePath = [picture for picture in os.listdir(folderPath)
                   if re.findall(r"\.png$|\.jpg$|\.JPG$|\.PNG$|\.JPEG$",picture)]
  
    for picture in picturePath:
      timeMemory = time()
      count = count + 1
      #load image file
      image = cv2.imread(folderPath + "/" + picture)
      #resize and crop<commit_msg>log was modified<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import os, sys, re
import random
from time import time

import cv2
import numpy as np

import chainer
from chainer import cuda
import chainer.functions as F
import overwrite
from chainer.functions import caffe

def getNeuralCode(directory, layer="fc6", gpu = -1):

  model = "bvlc_alexnet.caffemodel"
  #use alexnet
  print('alexnet is being loaded!')
  #calculate load time
  timeMemory = time()
  func = caffe.CaffeFunction(model)
  print('alexnet was loaded!')
  print('It took ' + str(int(time() - timeMemory)) + " secondes")

  #gpu mode
  if gpu >= 0:
    cuda.init(gpu)
    func.to_gpu()

  in_size = 227
  mean_image = np.load("ilsvrc_2012_mean.npy")

  # return neural code
  print("neural code is extraced from layer " + layer)
  def neuralCode(x):
    y, = func(inputs={'data': x}, outputs=[layer], train=False)
    return y.data[0]

  cropwidth = 256 - in_size
  start = cropwidth // 2
  stop = start + in_size
  mean_image = mean_image[:, start:stop, start:stop].copy()
  target_shape = (256, 256)
  output_side_length=256

  numPic = 0
  #count pictures
  for folderPath in directory:
    #search pictures
    picturePath = [picture for picture in os.listdir(folderPath)
                   if re.findall(r"\.png$|\.jpg$|\.JPG$|\.PNG$|\.JPEG$",picture)]
    print("you have " + str(len(picturePath)) + " pictures in " + folderPath)
    numPic = numPic + len(picturePath)
  
  print("you have totally " + str(numPic) + " pictures")
  answer = {}
  count = 0
  for folderPath in directory:
    #search pictures
    picturePath = [picture for picture in os.listdir(folderPath)
                   if re.findall(r"\.png$|\.jpg$|\.JPG$|\.PNG$|\.JPEG$",picture)]
  
    for picture in picturePath:
      timeMemory = time()
      count = count + 1
      #load image file
      image = cv2.imread(folderPath + "/" + picture)
      #resize and crop
{'fraction_non_alphanumeric': 0.12828282828282828, 'fraction_numerical': 0.015151515151515152, 'mean_word_length': 3.231837606837607, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6736828', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1322, 'n_words': 369}