<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from datetime import datetime
import os
import time
import copy
import sys
import numpy as np
import importlib
import itertools
import argparse
import math
import json
import random
#from guuker import prt

SEED = 727

DATA_ROOT = '/data1/deepinsight/carvn'

id2images = {}

train_image_dir = os.path.join(DATA_ROOT, 'train_hq')
for image in os.listdir(train_image_dir):
    image = image.strip()
    if not image.endswith('.jpg'):
        continue
    basename = image.split('.')[0]
    image_id, pos = basename.split('_')
    if image_id not in id2images:
        id2images[image_id] = 1
  #id2images[image_id].append(os.path.join(train_image_dir, image))
  #id2images[image_id].append( (image_id, pos) )

id2images = sorted(id2images.items(), key = lambda x: x[0])
random.Random(SEED).shuffle(id2images)
train_f = open('train.lst', 'w')
val_f = open('val.lst', 'w')
val_count = len(id2images)//20
for i in xrange(len(id2images)):
    #f = val_f if i<val_count else train_f
    image_id = id2images[i][0]
    idxs = range(1, 17)
    random.Random(SEED).shuffle(idxs)
    for idx in idxs:
        data_image = os.path.join(DATA_ROOT, 'train_hq', "%s_%02d.jpg" % (image_id, idx))
        mask_image = os.path.join(DATA_ROOT, 'train_masks', "%s_%02d_mask.gif" % (image_id, idx))
        if i < val_count:
            val_f.write("%d\t%s\t%s\n" % (0, data_image, mask_image))
        else:
            train_f.write("%d\t%s\t%s\n" % (0, data_image, mask_image))

train_f.close()
val_f.close()

<commit_msg>use 1 car as val<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from datetime import datetime
import os
import time
import copy
import sys
import numpy as np
import importlib
import itertools
import argparse
import math
import json
import random
#from guuker import prt

SEED = 727

DATA_ROOT = '/data1/deepinsight/carvn'

val_idmap = {'120c1dc7b116': 1}

id2images = {}

train_image_dir = os.path.join(DATA_ROOT, 'train_hq')
for image in os.listdir(train_image_dir):
    image = image.strip()
    if not image.endswith('.jpg'):
        continue
    basename = image.split('.')[0]
    image_id, pos = basename.split('_')
    if image_id not in id2images:
        id2images[image_id] = 1
  #id2images[image_id].append(os.path.join(train_image_dir, image))
  #id2images[image_id].append( (image_id, pos) )

id2images = sorted(id2images.items(), key = lambda x: x[0])
random.Random(SEED).shuffle(id2images)
train_f = open('train.lst', 'w')
val_f = open('val.lst', 'w')
val_count = len(id2images)//20
for i in xrange(len(id2images)):
    #f = val_f if i<val_count else train_f
    image_id = id2images[i][0]
    idxs = range(1, 17)
    random.Random(SEED).shuffle(idxs)
    for idx in idxs:
        data_image = os.path.join(DATA_ROOT, 'train_hq', "%s_%02d.jpg" % (image_id, idx))
        mask_image = os.path.join(DATA_ROOT, 'train_masks', "%s_%02d_mask.gif" % (image_id, idx))
        if image_id in val_idmap or (len(val_idmap)==0 and i < val_count):
            val_f.write("%d\t%s\t%s\n" % (0, data_image, mask_image))
        else:
            train_f.write("%d\t%s\t%s\n" % (0, data_image, mask_image))

train_f.close()
val_f.close()


{'fraction_non_alphanumeric': 0.13409158934450763, 'fraction_numerical': 0.021251122418437594, 'mean_word_length': 3.9584569732937687, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5088945', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1257, 'n_words': 293}