<commit_before>#!/usr/bin/env python
"""
Copyright (c) 2015, <NAME>

This program translate sloth json annotations to
FindYourSister json annotations.

Usage: ./gen_labels.py [gen_labels.json]

This program generates json file that FindYourSister can understand
from other files. They are specified in a json configuration file.

The default configuration file is `gen_labels.json` in current directory.

If you want to display the json info in screen rather than write to
a file, please set `outputJsonFile` to Null in configuration.
"""

import sys
import json
import re

# Global variables
GROUP_TYPES = set(["train", "validate", "validation", "test"])
CONFIG_FILE = "gen_labels.json"


def main():
    global CONFIG_FILE

    if len(sys.argv) == 2:
        CONFIG_FILE = sys.argv[2]

    configDoc = open(CONFIG_FILE, "r")
    configJson = json.load(configDoc)
    print json.JSONEncoder(indent = 4).encode(configJson)

    # Parse config file
    outfile = configJson[u"outputJsonFile"]

    labelFiles = []
    numLabelFiles = len(configJson[u"imgLabelFiles"])
    i = 0
    while i < numLabelFiles:
        labelFiles.append(configJson[u"imgLabelFiles"][i][u"filename"])
        i += 1

    infoFiles = []
    numInfoFiles = len(configJson[u"imgInfoFiles"])
    i = 0
    while i < numInfoFiles:
        infoFiles.append(configJson[u"imgInfoFiles"][i][u"filename"])
        i += 1

    # Setup the root node of output
    rootNode = setupJsonRoot()

    # Setup the image info database
    infoNode = setupJsonRoot()
    i = 0
    while i < numInfoFiles:
        addInfo(infoNode, infoFiles[i])
        i += 1
    print infoNode

    # TODO: Add each label file into the json structure


def setupJsonRoot():
    root = {}
    root[u"train"] = []
    root[u"validate"] = []
    root[u"test"] = []
    return root


def addInfo(node, filepath):
    fd = open(filepath, "r")
    data = json.load(fd)
    group = None
    if u"train" in data.keys():
        group = u"train"
    elif u"validate" in data.keys():
        group = u"validate"
    elif u"test" in data.keys():
        group = u"test"

    if group == None:
        return
    else:
        # Add all image info into database
        for item in data[group]:
            node[group].append(item)


def getDirectFolder(path):
    """
    This function returns the direct folder name of a file.
    It is used to match files given different types of path.
    """
    pattern = "/\w+"
    result = re.findall(pattern, path)
    return result[-1][1:]  # [1:] remove the '/' char at the beginning


if __name__ == "__main__":
    main()
<commit_msg>generate all-in-one info, tested<commit_after>#!/usr/bin/env python
"""
Copyright (c) 2015, <NAME>

This program translate sloth json annotations to
FindYourSister json annotations.

Usage: ./gen_labels.py [gen_labels.json]

This program generates json file that FindYourSister can understand
from other files. They are specified in a json configuration file.

The default configuration file is `gen_labels.json` in current directory.

If you want to display the json info in screen rather than write to
a file, please set `outputJsonFile` to null in configuration.
"""

import sys
import json
from os.path import split
import re

# Global variables
GROUP_TYPES = set(["train", "validate", "validation", "test"])
CONFIG_FILE = "gen_labels.json"


def main():
    global CONFIG_FILE

    if len(sys.argv) == 2:
        CONFIG_FILE = sys.argv[2]

    configDoc = open(CONFIG_FILE, "r")
    configJson = json.load(configDoc)
    # print json.JSONEncoder(indent = 4).encode(configJson)

    # Parse config file
    outfile = configJson[u"outputJsonFile"]

    labelFiles = []
    numLabelFiles = len(configJson[u"imgLabelFiles"])
    i = 0
    while i < numLabelFiles:
        labelFiles.append(configJson[u"imgLabelFiles"][i][u"filename"])
        i += 1

    infoFiles = []
    numInfoFiles = len(configJson[u"imgInfoFiles"])
    i = 0
    while i < numInfoFiles:
        infoFiles.append(configJson[u"imgInfoFiles"][i][u"filename"])
        i += 1

    # Setup the root node of output
    rootNode = setupJsonRoot()

    # Setup the image info database
    infoNode = setupJsonRoot()
    i = 0
    while i < numInfoFiles:
        addInfo(infoNode, infoFiles[i])
        i += 1
    # print infoNode

    # Process each label file
    i = 0
    while i < numLabelFiles:
        processLabels(rootNode, labelFiles[i], infoNode)
        i += 1

    # Write the results
    if outfile is None:
        print json.JSONEncoder(indent = 4).encode(rootNode)
    else:
        out = open(outfile, "w")
        out.write(json.JSONEncoder(indent = 4).encode(rootNode))


def setupJsonRoot():
    root = {}
    root[u"train"] = []
    root[u"validate"] = []
    root[u"test"] = []
    return root


def addInfo(root, filepath):
    fd = open(filepath, "r")
    data = json.load(fd)
    group = None
    if u"train" in data.keys():
        group = u"train"
    elif u"validate" in data.keys():
        group = u"validate"
    elif u"test" in data.keys():
        group = u"test"

    if group is None:
        return
    else:
        # Add all image info into database
        for item in data[group]:
            root[group].append(item)


def processLabels(root, filepath, refInfo):
    fd = open(filepath, "r")
    data = json.load(fd)
    # Each image is stored as a dict, collected in a list
    for img in data:
        processImage(root, img, refInfo)


def processImage(root, imgDict, refInfo):
    imgDir, imgFile = split(imgDict[u"filename"])
    imgDir = getDirectFolder(imgDir)
    group, node = findInfo(imgDir, imgFile, refInfo)
    objID = 0

    if group is not None:
        node[u"objects"] = []
        for label in imgDict[u"annotations"]:
            obj = {}
            obj[u"name"] = label[u"class"]
            obj[u"id"] = objID
            objID += 1
            obj[u"region"] = {}
            obj[u"region"][u"xmin"] = int(label[u"x"])
            obj[u"region"][u"xmax"] = int(label[u"x"] + label[u"width"])
            obj[u"region"][u"ymin"] = int(label[u"y"])
            obj[u"region"][u"ymax"] = int(label[u"y"] + label[u"height"])
            node[u"objects"].append(obj)
        root[group].append(node)


def findInfo(imgDir, imgFile, refInfo):
    """
    Helper function

    Returns (group, refInfo_node) if query image is in refInfo.
    Otherwise returns (None, None)
    """
    if u"train" in refInfo:
        for info in refInfo[u"train"]:
            if info[u"filename"] == imgFile and \
                    getDirectFolder(info[u"folder"]) == imgDir:
                return (u"train", info)
    if u"validate" in refInfo:
        for info in refInfo[u"train"]:
            if info[u"filename"] == imgFile and \
                    getDirectFolder(info[u"folder"]) == imgDir:
                return (u"validate", info)
    if u"test" in refInfo:
        for info in refInfo[u"train"]:
            if info[u"filename"] == imgFile and \
                    getDirectFolder(info[u"folder"]) == imgDir:
                return (u"test", info)
    return (None, None)


def getDirectFolder(path):
    """
    Helper function

    This function returns the direct folder name of a file.
    It is used to match files given different types of path.
    """
    pattern = "/\w+"
    result = re.findall(pattern, path)
    return result[-1][1:]  # [1:] remove the '/' char at the beginning


if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.12083054253181513, 'fraction_numerical': 0.005090421969189551, 'mean_word_length': 2.677832512315271, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '616201', 'n_tokens_mistral': 2377, 'n_tokens_neox': 2297, 'n_words': 693}