<commit_before>#!/usr/bin/env python

import sys
import os
import logging
import argparse
import json
from ece2cmor3 import ece2cmorlib,taskloader,cmor_source,cmor_utils

# Logging configuration
logging.basicConfig(level=logging.DEBUG)

# Logger construction
log = logging.getLogger(__name__)


# Converts the input fortran variable namelist file to json
def write_varlist(targets,opath):
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    tgtdict = dict([k,[t.variable for t in v]] for k,v in tgtgroups.iteritems())
    with open(opath,'w') as ofile:
        json.dump(tgtdict,ofile,indent = 4,separators = (',', ': '))
    logging.info("File %s written" % opath)


def write_varlist_ascii(targets,opath):
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    ofile = open(opath,'w')
    ofile.write('{:10} {:40} {:20} {:95} {:20} {} {}'.format('table', 'dimensions', 'variable', 'long_name', 'comment_author', 'comment', '\n'))
    for k,vlist in tgtgroups.iteritems():
        ofile.write('{}'.format('\n'))
        for tgtvar in vlist:
            ofile.write('{:10} {:40} {:20} {:95} {:20} {} {}'.format(tgtvar.table, getattr(tgtvar,"dimensions","unknown"), tgtvar.variable, getattr(tgtvar,"long_name","unknown"), getattr(tgtvar,"comment_author",""), getattr(tgtvar,"ignore_comment",""), '\n'))
    ofile.close()


def write_varlist_excel(targets,opath):
    import xlsxwriter
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    workbook = xlsxwriter.Workbook(opath)
    worksheet = workbook.add_worksheet()

    worksheet.set_column('A:A', 10)  # Adjust the column width of column A
    worksheet.set_column('B:B', 35)  # Adjust the column width of column B
    worksheet.set_column('C:C', 15)  # Adjust the column width of column C
    worksheet.set_column('D:D', 80)  # Adjust the column width of column D
    worksheet.set_column('E:E', 15)  # Adjust the column width of column G
    worksheet.set_column('F:F', 80)  # Adjust the column width of column E
    worksheet.set_column('G:G', 15)  # Adjust the column width of column F

    bold = workbook.add_format({'bold': True})   # Add a bold format

    worksheet.write(0, 0, 'Table', bold)
    worksheet.write(0, 1, 'Dimension format of variable', bold)
    worksheet.write(0, 2, 'variable', bold)
    worksheet.write(0, 3, 'variable long name', bold)
    worksheet.write(0, 4, 'link', bold)
    worksheet.write(0, 5, 'comment', bold)
    worksheet.write(0, 6, 'comment author', bold)
    
    row_counter = 1
    for k,vlist in tgtgroups.iteritems():
        worksheet.write(row_counter, 0, '')
        row_counter += 1
        for tgtvar in vlist:
            worksheet.write(row_counter, 0, tgtvar.table)
            worksheet.write(row_counter, 1, getattr(tgtvar,"dimensions","unknown"))
            worksheet.write(row_counter, 2, tgtvar.variable)
            worksheet.write(row_counter, 3, getattr(tgtvar,"long_name","unknown"))
            worksheet.write(row_counter, 4, '=HYPERLINK("' + 'http://clipc-services.ceda.ac.uk/dreq/u/' + getattr(tgtvar,"long_name","unknown").replace(' ', '_') + '","' + tgtvar.variable + '")')
            worksheet.write(row_counter, 5, getattr(tgtvar,"ignore_comment",""))
            worksheet.write(row_counter, 6, getattr(tgtvar,"comment_author",""))
            row_counter += 1
    workbook.close()


# Main program
def main():

    parser = argparse.ArgumentParser(description = "Validate input variable list against CMIP tables")
    parser.add_argument("--vars",   metavar = "FILE",   type = str, required = True, help = "File (json|f90 namelist|xlsx) containing cmor variables (Required)")
    parser.add_argument("--tabdir", metavar = "DIR",    type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory")
    parser.add_argument("--tabid",  metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string")
    parser.add_argument("--output", metavar = "FILE",   type = str, default = None, help = "Output path to write variables to")
    parser.add_argument("-v", "--verbose", action = "store_true", default = False, help = "Write xlsx and ASCII files with verbose output")
    parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables")
    parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables")

    args = parser.parse_args()

    # Initialize ece2cmor:
    ece2cmorlib.initialize(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid)

    # Fix conflicting flags
    procatmos,prococean = not args.oce,not args.atm
    if(not procatmos and not prococean):
        procatmos,prococean = True,True

    # Load the variables as task targets:
    loadedtargets,ignoredtargets,missingtargets = taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean)

    if(args.output):
        ofile,fext = os.path.splitext(args.output)
        write_varlist(loadedtargets,ofile + ".available.json")
        if(args.verbose):
            write_varlist_ascii(loadedtargets,ofile + ".available.txt")
            write_varlist_ascii(ignoredtargets,ofile + ".ignored.txt")
            write_varlist_ascii(missingtargets,ofile + ".missing.txt")

            write_varlist_excel(loadedtargets,ofile + ".available.xlsx")
            write_varlist_excel(ignoredtargets,ofile + ".ignored.xlsx")
            write_varlist_excel(missingtargets,ofile + ".missing.xlsx")

    # Finishing up
    ece2cmorlib.finalize()

if __name__ == "__main__":
    main()
<commit_msg>Adding the extensive CMOR variable description to the excel files produced by checvars.py #76<commit_after>#!/usr/bin/env python

import sys
import os
import logging
import argparse
import json
from ece2cmor3 import ece2cmorlib,taskloader,cmor_source,cmor_utils

# Logging configuration
logging.basicConfig(level=logging.DEBUG)

# Logger construction
log = logging.getLogger(__name__)


# Converts the input fortran variable namelist file to json
def write_varlist(targets,opath):
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    tgtdict = dict([k,[t.variable for t in v]] for k,v in tgtgroups.iteritems())
    with open(opath,'w') as ofile:
        json.dump(tgtdict,ofile,indent = 4,separators = (',', ': '))
    logging.info("File %s written" % opath)


def write_varlist_ascii(targets,opath):
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    ofile = open(opath,'w')
    ofile.write('{:10} {:40} {:20} {:95} {:20} {} {}'.format('table', 'dimensions', 'variable', 'long_name', 'comment_author', 'comment', '\n'))
    for k,vlist in tgtgroups.iteritems():
        ofile.write('{}'.format('\n'))
        for tgtvar in vlist:
            ofile.write('{:10} {:40} {:20} {:95} {:20} {} {}'.format(tgtvar.table, getattr(tgtvar,"dimensions","unknown"), tgtvar.variable, getattr(tgtvar,"long_name","unknown"), getattr(tgtvar,"comment_author",""), getattr(tgtvar,"ignore_comment",""), '\n'))
    ofile.close()


def write_varlist_excel(targets,opath):
    import xlsxwriter
    tgtgroups = cmor_utils.group(targets,lambda t:t.table)
    workbook = xlsxwriter.Workbook(opath)
    worksheet = workbook.add_worksheet()

    worksheet.set_column('A:A', 10)  # Adjust the column width of column A
    worksheet.set_column('B:B', 35)  # Adjust the column width of column B
    worksheet.set_column('C:C', 15)  # Adjust the column width of column C
    worksheet.set_column('D:D', 80)  # Adjust the column width of column D
    worksheet.set_column('E:E',  3)  # Adjust the column width of column G
    worksheet.set_column('F:F', 80)  # Adjust the column width of column E
    worksheet.set_column('G:G', 15)  # Adjust the column width of column F
    worksheet.set_column('H:H',300)  # Adjust the column width of column G

    bold = workbook.add_format({'bold': True})   # Add a bold format

    worksheet.write(0, 0, 'Table', bold)
    worksheet.write(0, 1, 'Dimension format of variable', bold)
    worksheet.write(0, 2, 'variable', bold)
    worksheet.write(0, 3, 'variable long name', bold)
    worksheet.write(0, 4, 'link', bold)
    worksheet.write(0, 5, 'comment', bold)
    worksheet.write(0, 6, 'comment author', bold)
    worksheet.write(0, 7, 'extensive variable description', bold)
    
    row_counter = 1
    for k,vlist in tgtgroups.iteritems():
        worksheet.write(row_counter, 0, '')
        row_counter += 1
        for tgtvar in vlist:
            worksheet.write(row_counter, 0, tgtvar.table)
            worksheet.write(row_counter, 1, getattr(tgtvar,"dimensions","unknown"))
            worksheet.write(row_counter, 2, tgtvar.variable)
            worksheet.write(row_counter, 3, getattr(tgtvar,"long_name","unknown"))
            worksheet.write(row_counter, 4, '=HYPERLINK("' + 'http://clipc-services.ceda.ac.uk/dreq/u/' + getattr(tgtvar,"long_name","unknown").replace(' ', '_').lower() + '.html","web")')
           #worksheet.write(row_counter, 4, '=HYPERLINK("' + 'http://clipc-services.ceda.ac.uk/dreq/u/' + getattr(tgtvar,"long_name","unknown").replace(' ', '_').lower() + '.html","' + tgtvar.variable + '")')
            worksheet.write(row_counter, 5, getattr(tgtvar,"ignore_comment",""))
            worksheet.write(row_counter, 6, getattr(tgtvar,"comment_author",""))
            worksheet.write(row_counter, 7, getattr(tgtvar,"comment","unknown"))
            row_counter += 1
    workbook.close()


# Main program
def main():

    parser = argparse.ArgumentParser(description = "Validate input variable list against CMIP tables")
    parser.add_argument("--vars",   metavar = "FILE",   type = str, required = True, help = "File (json|f90 namelist|xlsx) containing cmor variables (Required)")
    parser.add_argument("--tabdir", metavar = "DIR",    type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory")
    parser.add_argument("--tabid",  metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string")
    parser.add_argument("--output", metavar = "FILE",   type = str, default = None, help = "Output path to write variables to")
    parser.add_argument("-v", "--verbose", action = "store_true", default = False, help = "Write xlsx and ASCII files with verbose output")
    parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables")
    parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables")

    args = parser.parse_args()

    # Initialize ece2cmor:
    ece2cmorlib.initialize(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid)

    # Fix conflicting flags
    procatmos,prococean = not args.oce,not args.atm
    if(not procatmos and not prococean):
        procatmos,prococean = True,True

    # Load the variables as task targets:
    loadedtargets,ignoredtargets,missingtargets = taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean)

    if(args.output):
        ofile,fext = os.path.splitext(args.output)
        write_varlist(loadedtargets,ofile + ".available.json")
        if(args.verbose):
            write_varlist_ascii(loadedtargets,ofile + ".available.txt")
            write_varlist_ascii(ignoredtargets,ofile + ".ignored.txt")
            write_varlist_ascii(missingtargets,ofile + ".missing.txt")

            write_varlist_excel(loadedtargets,ofile + ".available.xlsx")
            write_varlist_excel(ignoredtargets,ofile + ".ignored.xlsx")
            write_varlist_excel(missingtargets,ofile + ".missing.xlsx")

    # Finishing up
    ece2cmorlib.finalize()

if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.14177343223196223, 'fraction_numerical': 0.012811867835468645, 'mean_word_length': 3.98320033599328, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '945967', 'n_tokens_mistral': 4088, 'n_tokens_neox': 3870, 'n_words': 869}