<commit_before>from os import listdir, remove, makedirs
from os.path import join, exists
from shutil import copy2 as cp

datadir = './dump'
new_dir = './new_dump'
end = 'rgb.png'

from_dict = {}
to_dict = {}


#form from_dict
f_d = join(datadir,'dumps')
for f in listdir(f_d):
  if(not f.startswith('.')):
    p_d = {}
    for p in listdir(join(f_d,f)):
      if not p.endswith('.png'):
        continue
      trid,pid,_ = p.rsplit('_')
      curr = p_d.get(trid)
      if curr:
        curr.append(pid)
        p_d[trid] = curr
      else:
        p_d[trid] = [pid]
    from_dict[f] = p_d

#form to_dict
d = {'4p':['backleft','backright', 'frontleft', 'frontright'], '4x':['back', 'front', 'left', 'right']}
to_dict = {'4p':{},'4x':{}}

#create all dirs we need
for k in d:
  for el in d[k]:
    path = join(new_dir, k, el)
    if not exists(path):
      makedirs(path)

for k in d:
    for el in d[k]:
        to_dict[k][el] = {}
        p = join(datadir, k, el)
        for f in listdir(p):
            if(not f.startswith('.')):
                name,trid,pid,_ = f.rsplit('_')
                if to_dict.get(k).get(el).get(name,0) == 0:
                    to_dict[k][el][name] = {}
                if to_dict.get(k).get(el).get(name).get(trid, 0) == 0:
                    to_dict[k][el][name][trid] = [pid]
                else:
                    to_dict[k][el][name][trid].append(pid)

#iterate inside to_dict and copy all pics from from_dict that are inside
#min and max inside every folder with respect to track id and name

def crem(t,a,n,tid,pid,end):
    f_name = join(datadir, 'dumps', n, "_".join([tid, pid, end]))
    t_name = join(new_dir, t, a, "_".join([n, tid, pid, end]))
    csvf_name = join(datadir, 'dumps', n, "_".join([tid, pid, 'd.csv']))
    csvt_name = join(new_dir, t, a, "_".join([n, tid, pid, 'd.csv']))
    #print(f_name, t_name)
    cp(f_name, t_name)
    cp(csvf_name, csvt_name)
    from_dict[n][tid].remove(pid)
    #remove(f_name)

for t in to_dict:
    for a in to_dict[t]:
        for n in to_dict[t][a]:
            for tid in to_dict[t][a][n]:
                pids = to_dict[t][a][n][tid]
                min_pid = min(pids)
                max_pid = max(pids)
                f_pids = from_dict.get(n, None)
                if f_pids:
                    f_pids = from_dict[n].get(tid, None)
                    if f_pids:
                        for pid in f_pids:
                           if min_pid<=pid<=max_pid:
                               crem(t, a, n, tid, pid, end)
<commit_msg>Fixed data labeling script. Copying only same names now.<commit_after>from os import listdir, remove, makedirs
from os.path import join, exists
from shutil import copy2 as cp

datadir = '/work/kurin/spencer_data/dump'
new_dir = '/work/kurin/spencer_data/new_dump'
end = 'rgb.png'

from_dict = {}
to_dict = {}


#form from_dict
f_d = datadir
f_d = join(datadir,'dumps')
for f in listdir(f_d):
  if(not f.startswith('.')):
    p_d = {}
    for p in listdir(join(f_d,f)):
      if not p.endswith('.png'):
        continue
      trid,pid,_ = p.rsplit('_')
      curr = p_d.get(trid)
      if curr:
        curr.append(pid)
        p_d[trid] = curr
      else:
        p_d[trid] = [pid]
    from_dict[f] = p_d

#form to_dict
d = {'4p':['backleft','backright', 'frontleft', 'frontright'], '4x':['back', 'front', 'left', 'right']}
to_dict = {'4p':{},'4x':{}}

#create all dirs we need
for k in d:
  for el in d[k]:
    path = join(new_dir, k, el)
    if not exists(path):
      makedirs(path)

for k in d:
    for el in d[k]:
        to_dict[k][el] = {}
        p = join(datadir, k, el)
        for f in listdir(p):
            if(not f.startswith('.')):
                name,trid,pid,_ = f.rsplit('_')
                if to_dict.get(k).get(el).get(name,0) == 0:
                    to_dict[k][el][name] = {}
                if to_dict.get(k).get(el).get(name).get(trid, 0) == 0:
                    to_dict[k][el][name][trid] = [pid]
                else:
                    to_dict[k][el][name][trid].append(pid)

#iterate in to_dict and copy pics from from_dict with the same name, tid and pid

def crem(t,a,n,tid,pid,end):
    f_name = join(datadir, 'dumps', n, "_".join([tid, pid, end]))
    t_name = join(new_dir, t, a, "_".join([n, tid, pid, end]))
    #csvf_name = join(datadir, 'dumps', n, "_".join([tid, pid, 'd.csv']))
    #csvt_name = join(new_dir, t, a, "_".join([n, tid, pid, 'd.csv']))
    cp(f_name, t_name)
    #cp(csvf_name, csvt_name)
    #remove(f_name)

print("Generating in progress...")
n_im = 0
for t in to_dict:
    for a in to_dict[t]:
        for n in to_dict[t][a]:
            for tid in to_dict[t][a][n]:
                pids = to_dict[t][a][n][tid]
                if pids is not None:
                  for p in pids:
                    if p in from_dict[n].get(tid, []):
                      crem(t,a,n,tid,p,end)
                      n_im+=1
print "Finished. Copied %s images" % (n_im)

{'fraction_non_alphanumeric': 0.16740710823909533, 'fraction_numerical': 0.004038772213247173, 'mean_word_length': 2.154777070063694, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3361524', 'n_tokens_mistral': 1959, 'n_tokens_neox': 1908, 'n_words': 463}