<commit_before>import cv2, tqdm
import numpy as np

def load_data():
  X, Y = {}, {}
  for set in ["train", "valid"]:
    X[set], Y[set] = [[], []], []
    with open("../data/" + set + ".txt") as file:
      for line in tqdm.tqdm(file.readlines()):
        elements = line.strip().split()
        prefix, a, b = map(int, elements[:3])
        score = float(elements[3])
        if score >= 0.3 and score <= 0.7:
          continue
        for i, x in enumerate([a, b]):
          image = cv2.imread("../data/images/{:06d}-{:02d}.JPG".format(prefix, x))
          width, height = image.shape[:2]
          if width < height:
            width = int(224. * width / height)
            height = 224
          else:
            height = int(224. * height / width)
            width = 224
          image = cv2.resize(image, (height, width)).astype(np.float32)
          image = np.pad(image, ((0, 224 - width), (0, 224 - height), (0, 0)), mode = "constant", constant_values = 0)
          X[set][i].append(image)
        if score < 0.5:
          Y[set].append([1, 0])
        else:
          Y[set].append([0, 1])
    X[set][0], X[set][1], Y[set] = map(np.array, [X[set][0], X[set][1], Y[set]])
  return X, Y<commit_msg>fix crucial bug: wrong input scale<commit_after>import cv2, tqdm
import numpy as np

def load_data():
  X, Y = {}, {}
  for set in ["train", "valid"]:
    X[set], Y[set] = [[], []], []
    with open("../data/" + set + ".txt") as file:
      for line in tqdm.tqdm(file.readlines()):
        elements = line.strip().split()
        prefix, a, b = map(int, elements[:3])
        score = float(elements[3])
        if score >= 0.3 and score <= 0.7:
          continue
        for i, x in enumerate([a, b]):
          image = cv2.imread("../data/images/{:06d}-{:02d}.JPG".format(prefix, x))
          width, height = image.shape[:2]
          if width < height:
            width = int(224. * width / height)
            height = 224
          else:
            height = int(224. * height / width)
            width = 224
          image = cv2.resize(image, (height, width)).astype(np.float32)
          image = np.pad(image, ((0, 224 - width), (0, 224 - height), (0, 0)), mode = "constant", constant_values = 0)
          X[set][i].append(image / 255.)
        if score < 0.5:
          Y[set].append([1, 0])
        else:
          Y[set].append([0, 1])
    X[set][0], X[set][1], Y[set] = map(np.array, [X[set][0], X[set][1], Y[set]])
  return X, Y
{'fraction_non_alphanumeric': 0.19374492282696995, 'fraction_numerical': 0.041023558082859464, 'mean_word_length': 1.9963503649635037, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4170676', 'n_tokens_mistral': 906, 'n_tokens_neox': 848, 'n_words': 223}