<commit_before>#!/usr/bin/env python

import os
import sys
import logging
import f90nml
import json
import argparse
import cmor_utils
import cmor_source
import cmor_target
import cmor_task
import ece2cmor
import namloader
import jsonloader

# This script reads all variables from the fortran namelists in the resources and
# creates tasks for them. Then these tasks are serialized to the preferred json
# format for later use by ece2cmor. This scripts can be executed if the fortran
# namelist files are more complete than the json files and the latter need syncing.

logging.basicConfig(level=logging.WARNING)

# Converts the input fortran parameter namelist file to json
def convert_parlist(inputfile,outputfile):
    ifsparlist = f90nml.read(inputfile)
    pardictlist = ifsparlist.get("parameter")
    parlist = [p["out_name"] for p in pardictlist]
    targets = []
    for p in parlist:
        t = ece2cmor.get_cmor_target(p)
        if(isinstance(t,cmor_target.cmor_target)):
            targets.append(t)
        else:
            targets.extend(t)
    namloader.load_targets(targets)
    taskgroups = cmor_utils.group(ece2cmor.tasks,lambda t:t.target.variable)
    dictlist = map(makedict,[v[0] for (k,v) in taskgroups.iteritems()])
    with open(outputfile,'w') as ofile:
        json.dump(dictlist,ofile,indent = 4,separators = (',', ': '))

# Creates a ece2cmor-compliant dictionary for the given task
def makedict(task):
    result = {}
    result[jsonloader.json_source_key] = task.source.var_id if isinstance(task.source,cmor_source.nemo_source) else str(task.source.get_grib_code())
    result[jsonloader.json_target_key] = task.target.variable
    if(isinstance(task.source,cmor_source.nemo_source)):
        result[jsonloader.json_grid_key] = task.source.grid()
    if(hasattr(task.source,cmor_source.expression_key)):
        result[cmor_source.expression_key] = getattr(task.source,cmor_source.expression_key)
    if(hasattr(task,cmor_task.conversion_key)):
        result[cmor_task.conversion_key] = getattr(task,cmor_task.conversion_key)
    return result<commit_msg>Added support for multiple targets in json creation script<commit_after>#!/usr/bin/env python

import os
import sys
import logging
import f90nml
import json
import argparse
import cmor_utils
import cmor_source
import cmor_target
import cmor_task
import ece2cmor
import namloader
import jsonloader

# This script reads all variables from the fortran namelists in the resources and
# creates tasks for them. Then these tasks are serialized to the preferred json
# format for later use by ece2cmor. This scripts can be executed if the fortran
# namelist files are more complete than the json files and the latter need syncing.

logging.basicConfig(level=logging.WARNING)

# Converts the input fortran parameter namelist file to json
def convert_parlist(inputfile,outputfile):
    ifsparlist = f90nml.read(inputfile)
    pardictlist = ifsparlist.get("parameter")
    parlist = [p["out_name"] for p in pardictlist]
    targets = []
    for p in parlist:
        t = ece2cmor.get_cmor_target(p)
        if(isinstance(t,cmor_target.cmor_target)):
            targets.append(t)
        else:
            targets.extend(t)
    namloader.load_targets(targets)
    taskgroups = cmor_utils.group(ece2cmor.tasks,lambda t:t.target.variable)
    dictlist = map(makedict,[v[0] for (k,v) in taskgroups.iteritems()])
    jsondata = reduce(lambda dlist,d:reduce_dictlist(dlist,d),dictlist)
    with open(outputfile,'w') as ofile:
        json.dump(jsondata,ofile,indent = 4,separators = (',', ': '))

# Joins multiple task entries (e.g. ua,ua4,ua7c,...) to a single entry
def reduce_dictlist(dictlist,newdict):
    if(not(isinstance(dictlist,list))):
        return reduce_dictlist([dictlist],newdict)
    for d in dictlist:
        if({k:v for k,v in d.iteritems() if k != jsonloader.json_target_key} == {k:v for k,v in newdict.iteritems() if k != jsonloader.json_target_key}):
            existingtarget = d[jsonloader.json_target_key]
            newtarget = newdict[jsonloader.json_target_key]
            if(isinstance(existingtarget,list)):
                existingtarget.append(newtarget)
            else:
                d[jsonloader.json_target_key] = [existingtarget,newtarget]
            return dictlist
    dictlist.append(newdict)
    return dictlist

# Creates a ece2cmor-compliant dictionary for the given task
def makedict(task):
    result = {}
    result[jsonloader.json_source_key] = task.source.var_id if isinstance(task.source,cmor_source.nemo_source) else str(task.source.get_grib_code())
    result[jsonloader.json_target_key] = task.target.variable
    if(isinstance(task.source,cmor_source.nemo_source)):
        result[jsonloader.json_grid_key] = task.source.grid()
    if(hasattr(task.source,cmor_source.expression_key)):
        result[cmor_source.expression_key] = getattr(task.source,cmor_source.expression_key)
    if(hasattr(task,cmor_task.conversion_key)):
        result[cmor_task.conversion_key] = getattr(task,cmor_task.conversion_key)
    return result
{'fraction_non_alphanumeric': 0.09385496937364157, 'fraction_numerical': 0.004742145820983995, 'mean_word_length': 4.207818930041152, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3056252', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1614, 'n_words': 381}