<commit_before>import os
import glob
import re
import argparse

parser = argparse.ArgumentParser("Convert text files into JSON")
parser.add_argument("--path", type=str, help="Path to the input\
                    data. Expects a train/val/test_words files with\
                    one line per image description. Additionally expects\
                    train/val/test_images files with one image filename\
                    per line.")
parser.add_argument("--name", type=str, help="A recognisable name for the\
                    dataset.")
args = parser.parse_args()

handle = open("%s/dataset.json" % args.path, "w")
handle.write('{"images":[')

sent_counter = 0
imgid = 0

splits = ['train', 'val', 'test']

for split in splits:
  sentences = open("%s/%s_words" % (args.path, split)).readlines()
  sentences = [x.replace("\n","") for x in sentences]
  images = open("%s/%s_images" % (args.path, split)).readlines()
  images = [x.replace("\n","") for x in images]

  localidx = 0
  for sent, image in zip(sentences, images):

    startx = re.sub(r".jpg","", image)
  
    sentidstr = ""
    localCounter = sent_counter
    sentidstr += "%d, " % localCounter
    sentidstr = sentidstr[:-2]
  
    handle.write('{"filename":"%s", ' % image)
    handle.write('"sentids":[%s], ' % sentidstr)
    handle.write('"imgid":%d, ' % imgid)
    handle.write('"split":"%s", ' % split)
    handle.write('"sentences":[')
  
    scontent = sent.split(" ")
    tokens_str = ''
    for w in scontent:
      tokens_str += '"%s", ' % w.replace('"','')
    tokens_str = tokens_str[:-2]
      
    handle.write('{"tokens":[%s], ' % tokens_str)
    handle.write('"raw":"%s .", ' % sent)
    handle.write('"imgid":%d, ' % imgid)
    handle.write('"sentid": %d}]}' % sent_counter)
    sent_counter += 1
 
    if split == "test":
      if localidx < len(images)-1:
        handle.write(", ")
      else:
        continue
    else:
      handle.write(", ")
  
    imgid +=1
    localidx += 1

handle.write('], "dataset": "%s"}' % args.name)

handle.close()<commit_msg>makejson now cleanly supports multiple input files per split<commit_after>import os
import glob
import re
import argparse

parser = argparse.ArgumentParser("Convert text files into JSON")
parser.add_argument("--path", type=str, help="Path to the input\
                    data. Expects a train/val/test_words files with\
                    one line per image description. Additionally expects\
                    train/val/test_images files with one image filename\
                    per line.")
parser.add_argument("--name", type=str, help="A recognisable name for the\
                    dataset.")
args = parser.parse_args()

handle = open("%s/dataset.json" % args.path, "w")
handle.write('{"images":[')

sent_counter = 0
imgid = 0

splits = ['train', 'val', 'test']

for split in splits:
  sentence_files = glob.glob("%s/%s.*" % (args.path, split))
  sentences = []
  for fname in sentence_files:
      f_sentences = open("%s" % (fname)).readlines()
      f_sentences = [x.replace("\n","") for x in f_sentences]
      sentences.append(f_sentences)

  images = open("%s/%s_images" % (args.path, split)).readlines()
  images = [x.replace("\n","") for x in images]

  localidx = 0
  for idx, image in enumerate(images):
    local_sentences = []
    for x in sentences:
        local_sentences.append(x[idx])

    # Build the string that holds the sentence ids. This will look like
    # 1,2,3,4,5, and will be transformed into [%s] below.
    sent_ids = ""
    local_counter = sent_counter
    for s in local_sentences:
        sent_ids += "%d, " % local_counter
        local_counter += 1
    sent_ids = sent_ids[:-2] # Get rid of the trailing ", "

    handle.write('{"filename":"%s", ' % image)
    handle.write('"imgid":%d, ' % imgid)
    handle.write('"sentences":[')

    for sidx, s in enumerate(local_sentences):
        split_sent = s.split(" ")
        tokens_str = ''
        for w in split_sent:
          tokens_str += '"%s", ' % w.replace('"','')
        tokens_str = tokens_str[:-2]

        handle.write('{"tokens":[%s], ' % tokens_str)
        s = s.replace('"', '') # BE CAREFUL, WE ARE THROWING AWAY SPEECH MARKS
        handle.write('"raw":"%s", ' % s)
        handle.write('"imgid":%d, ' % imgid)
        handle.write('"sentid": %d} ' % sent_counter)
        if sidx < len(local_sentences)-1:
            handle.write(", ")
        sent_counter += 1
    handle.write("], ")
    handle.write('"split":"%s", ' % split)
    handle.write('"sentids":[%s]}' % sent_ids)

    if split == "test":
      if localidx < len(images)-1:
        handle.write(", ")
      else:
        continue
    else:
      handle.write(", ")

    imgid +=1
    localidx += 1

handle.write('], "dataset": "%s"}' % args.name)

handle.close()
{'fraction_non_alphanumeric': 0.16910229645093947, 'fraction_numerical': 0.005219206680584551, 'mean_word_length': 2.6996138996138996, 'pattern_counts': {'":': 24, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6226909', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1550, 'n_words': 387}