<commit_before>#!/usr/bin/env python
"""
Saves all the Cerulean dataset into jams. The structure of the Cerulean
dataset is:

/Cerulean
    /annotator
        /gt
            /json files

Only section annotations are available.

"""

__author__ = "<NAME>"
__copyright__ = "Copyright 2014, Music and Audio Research Lab (MARL)"
__license__ = "GPL"
__version__ = "1.0"
__email__ = "<EMAIL>"

import argparse
import glob
import jams
import json
import logging
import os
import sys
import time


def fill_global_metadata(jam, json_file):
    """Fills the global metada into the JAMS jam."""
    # Open JSON
    f = open(json_file, "r")
    annot = json.load(f)
    basename = os.path.basename(json_file).replace(".json","")

    if "-" in basename:
        token = "-"
    elif "," in basename:
        token = ","

    # Assign Metadata
    jam.metadata.artist = basename.split(token)[0]
    jam.metadata.duration = -1 # In seconds
    jam.metadata.title = basename.split(token)[1]
    jam.metadata.md5 = "" #TODO
    jam.metadata.echonest_id = annot["TRID"]
    jam.metadata.mbid = "" #TODO
    jam.metadata.version = -1 #TODO

    f.close()

def fill_annoatation_metadata(annot, attribute, name):
    """Fills the annotation metadata."""
    annot.annotation_metadata.attribute = attribute
    annot.annotation_metadata.corpus = "Cerulean"
    annot.annotation_metadata.version = "1.0"
    annot.annotation_metadata.annotation_tools = "Sonic Visualizer"
    annot.annotation_metadata.annotation_rules = "TODO" #TODO
    annot.annotation_metadata.validation_and_reliability = "TODO" #TODO
    annot.annotation_metadata.origin = "Cerulean Mountain Trust"
    annot.annotation_metadata.annotator.name = name
    annot.annotation_metadata.annotator.email = "TODO" #TODO
    #TODO:
    #time = "TODO"

def fill_section_annotation(json_file, annot):
    """Fills the JAMS annot annotation given a json file."""

    # Annotation Metadata
    fill_annoatation_metadata(annot, "sections", json_file.split("/")[-3])
    
    # Open JSON
    f = open(json_file, "r")
    json_annot = json.load(f)

    # Convert to JAMS
    for i, json_section in enumerate(json_annot["sections"][:-1]):
        start_time = json_section["start"]
        end_time = json_annot["sections"][i+1]["start"]
        label = json_section["label"]
        function_label = json_section["opt"]

        section = annot.create_datapoint()
        section.start.value = float(start_time)
        section.start.confidence = 1.0
        section.end.value = float(end_time)
        section.end.confidence = 1.0
        section.label.value = function_label
        section.label.confidence = 1.0
        section.level = "function" # Only function level of annotation

        section = annot.create_datapoint()
        section.start.value = float(start_time)
        section.start.confidence = 1.0
        section.end.value = float(end_time)
        section.end.confidence = 1.0
        section.label.value = label
        section.label.confidence = 1.0
        section.level = "large_scale" # Only function level of annotation

    f.close()


def create_JAMS(json_files, out_file):
    """Creates a JAMS file given the Cerulean json files."""

    # New JAMS and annotation
    jam = jams.Jams()<commit_msg>Removed a few unuseful comments in the code.<commit_after>#!/usr/bin/env python
"""
Saves all the Cerulean dataset into jams. The structure of the Cerulean
dataset is:

/Cerulean
    /annotator
        /gt
            /json files

Only section annotations are available.

"""

__author__ = "<NAME>"
__copyright__ = "Copyright 2014, Music and Audio Research Lab (MARL)"
__license__ = "GPL"
__version__ = "1.0"
__email__ = "<EMAIL>"

import argparse
import glob
import jams
import json
import logging
import os
import sys
import time


def fill_global_metadata(jam, json_file):
    """Fills the global metada into the JAMS jam."""
    # Open JSON
    f = open(json_file, "r")
    annot = json.load(f)
    basename = os.path.basename(json_file).replace(".json","")

    if "-" in basename:
        token = "-"
    elif "," in basename:
        token = ","

    # Assign Metadata
    jam.metadata.artist = basename.split(token)[0]
    jam.metadata.duration = -1 # In seconds
    jam.metadata.title = basename.split(token)[1]
    jam.metadata.md5 = "" #TODO
    jam.metadata.echonest_id = annot["TRID"]
    jam.metadata.mbid = "" #TODO
    jam.metadata.version = -1 #TODO

    f.close()

def fill_annoatation_metadata(annot, attribute, name):
    """Fills the annotation metadata."""
    annot.annotation_metadata.attribute = attribute
    annot.annotation_metadata.corpus = "Cerulean"
    annot.annotation_metadata.version = "1.0"
    annot.annotation_metadata.annotation_tools = "Sonic Visualizer"
    annot.annotation_metadata.annotation_rules = "TODO" #TODO
    annot.annotation_metadata.validation_and_reliability = "TODO" #TODO
    annot.annotation_metadata.origin = "Cerulean Mountain Trust"
    annot.annotation_metadata.annotator.name = name
    annot.annotation_metadata.annotator.email = "TODO" #TODO
    #TODO:
    #time = "TODO"

def fill_section_annotation(json_file, annot):
    """Fills the JAMS annot annotation given a json file."""

    # Annotation Metadata
    fill_annoatation_metadata(annot, "sections", json_file.split("/")[-3])
    
    # Open JSON
    f = open(json_file, "r")
    json_annot = json.load(f)

    # Convert to JAMS
    for i, json_section in enumerate(json_annot["sections"][:-1]):
        start_time = json_section["start"]
        end_time = json_annot["sections"][i+1]["start"]
        label = json_section["label"]
        function_label = json_section["opt"]

        section = annot.create_datapoint()
        section.start.value = float(start_time)
        section.start.confidence = 1.0
        section.end.value = float(end_time)
        section.end.confidence = 1.0
        section.label.value = function_label
        section.label.confidence = 1.0
        section.level = "function" 

        section = annot.create_datapoint()
        section.start.value = float(start_time)
        section.start.confidence = 1.0
        section.end.value = float(end_time)
        section.end.confidence = 1.0
        section.label.value = label
        section.label.confidence = 1.0
        section.level = "large_scale"

    f.close()


def create_JAMS(json_files, out_file):
    """Creates a JAMS file given the Cerulean json files."""

    # New JAMS and annotation
    jam = jams.Jams()
{'fraction_non_alphanumeric': 0.11537276648182378, 'fraction_numerical': 0.00862600123228589, 'mean_word_length': 3.0836477987421382, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6248320', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2007, 'n_words': 482}