<commit_before>package org.agmip.acmo.util;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;

import org.agmip.util.MapUtil;
import org.agmip.ace.LookupCodes;
import org.agmip.ace.util.AcePathfinderUtil;
import org.agmip.dome.DomeUtil;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AcmoUtil {
    private static final Logger log = LoggerFactory.getLogger(AcmoUtil.class);
    /**
     * Do not instantiate
     */
    private AcmoUtil(){}


    /**
     * Write an ACMO file to the specified {@code outputPath} which includes
     * all the experiments in the {@code datapackage}.
     *
     * @param outputPath the directory to write the ACMO_meta.dat file.
     * @param datapackage a standard AgMIP package
     *
     */
    public static void writeAcmo(String outputPath, HashMap<String, Object> datapackage, String destModel) {
        try {
            // Make sure the outputPath exists
            File f = new File(outputPath);
            f.mkdirs();
            String fileName   = outputPath+File.separator+"ACMO_meta.dat";
            log.debug("Attempting to write {}", fileName);
            FileWriter fw     = new FileWriter(fileName);
            BufferedWriter bw = new BufferedWriter(fw);
            try {
                // First write the header
                bw.write(generateAcmoHeader());
                 // Then write the lines
                ArrayList<HashMap<String, Object>> experiments = MapUtil.getRawPackageContents(datapackage, "experiments");
                for (HashMap<String, Object> experiment : experiments) {
                    String acmoData = extractAcmoData(experiment, destModel);
                    log.debug("ACMO dataline: {}", acmoData);
                    bw.write(acmoData);
                    bw.write("\n");
                }
            } finally {
                // Finally close the file.
                log.debug("Attempting to close the ACMO file");
                bw.flush();
                bw.close();
                fw.close();
            }
        } catch (Exception ex) {
            log.error("Error writing ACMO_meta.dat: {}", ex.getMessage());
        }
    }

    /**
     * Extract a CSV-compatable ACMO line from a single experiment
     *
     * @param dataset a single AgMIP dataset
     *
     * @return ACMO compatible CSV line.
     */
    public static String extractAcmoData(HashMap<String, Object> dataset, String destModel) {
        ArrayList<String> acmoData = new ArrayList<String>();
        HashMap<String, Object> observed = MapUtil.getRawBucket(dataset, "observed");
        HashMap<String, String> events   = extractEventData(dataset, destModel);
        /**
         * Data to get:
         * wst_id (root)
         * soil_id (root)
         * fl_lat (root)
         * fl_lon (root)
         * crid (events)
         * cul_id (events)
         * cul_name (events)
         * sdat (root)
         * pdate (events)
         * hdate (observed)
         * hwah (observed)
         * cwah (observed)
         * ir_count (events)
         * ir_tot (events)
         * irop (events)
         * fe_count (events)
         * fen_tot (events)
         * fep_tot (events)
         * fek_tot (events)
         * omamt (events)
         * ti_count (events)
         * tiimp (events)
         * crop model (passed into writeAcmo())
         */

        String fieldOverlayString = MapUtil.getValueOr(dataset, "field_overlay", "");
        String seasonalStrategyString = MapUtil.getValueOr(dataset, "seasonal_strategy", "");
        HashMap<String, String> domeBase = new HashMap<String, String>();

        acmoData.add("*"); // Just an indication mark, this line stands alone
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "exname", ""))); // Will be generated by the database
        acmoData.add("1");
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "field_overlay", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "seasonal_strategy", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "exname", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "trt_name", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "clim_id", "0XXX")));
        acmoData.add("1");
        if (! seasonalStrategyString.equals("")) {
            domeBase = DomeUtil.unpackDomeName(seasonalStrategyString);
        } else if (! fieldOverlayString.equals("")){
            domeBase = DomeUtil.unpackDomeName(fieldOverlayString);
        }
        acmoData.add(quoteMe(MapUtil.getValueOr(domeBase, "region", ""))); // Region
        acmoData.add(quoteMe(MapUtil.getValueOr(domeBase, "stratum", ""))); // Stratum
        acmoData.add(MapUtil.getValueOr(domeBase, "rap_id", "")); // RAP ID
        acmoData.add(MapUtil.getValueOr(domeBase, "man_id", "")); // MAN ID
        acmoData.add("AgMIP"); // Institution
        acmoData.add(MapUtil.getValueOr(dataset, "rotation", "0"));
        acmoData.add(MapUtil.getValueOr(dataset, "wst_id", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "soil_id", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "fl_lat", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "fl_long", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "crid", "")));
        acmoData.add(MapUtil.getValueOr(events, "cul_id", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "cul_name", "")));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(dataset, "sdat", "")));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(events, "pdate", "")));
        acmoData.add(MapUtil.getValueOr(observed, "hwah", ""));
        acmoData.add(MapUtil.getValueOr(observed, "cwah", ""));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(observed, "hdate", "")));
        acmoData.add(MapUtil.getValueOr(events, "ir_count", ""));
        acmoData.add(MapUtil.getValueOr(events, "ir_tot", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "irop", "")));
        acmoData.add(MapUtil.getValueOr(events, "fe_count", ""));
        acmoData.add(MapUtil.getValueOr(events, "fen_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "fep_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "fek_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "omamt", ""));
        acmoData.add(MapUtil.getValueOr(events, "ti_count", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "tiimp", "")));
        acmoData.add(destModel.toUpperCase());
        return joinList(acmoData, ",");
    }

    /**
     * Generates the standard ACMO header (with all the variables defined)
     *
     * @return ACMO header
     */
    public static String generateAcmoHeader() {
        return "!,Experiment ID,,Field Overlay ID,Seasonal Strategy ID,\"Name of experiment, field test or survey\",Treatment Name,4-character Climate ID code,Climate replication number for multiple realizations of weather data (ask Alex),Region ID,Regional stratum identification number,RAP ID,\"Management regimen ID, for multiple management regimens per RAP\",Names of institutions involved in collection of field or survey data,\"Crop rotation indicator (=1 to indicate that this is a continuous, multi-year simulation, =0 for single year simulations)\",Weather station ID,Soil ID,Site Latitude,Site Longitude,Crop type (common name) ,Crop model-specific cultivar ID,Cultivar name,Start of simulation date,Planting date,\"Observed harvested yield, dry weight\",Observed total above-ground biomass at harvest,Observed harvest date,Total number of irrigation events,Total amount of irrigation,Type of irrigation application,Total number of fertilizer applications,Total N applied,Total P applied,Total K applied,Manure and applied oganic matter,Total number of tillage applications,\"Tillage type (hand, animal or mechanized)\",\"Short name of crop model used for simulations (e.g., DSSAT, APSIM, Aquacrop, STICS, etc.)\",Model name and version number of the crop model used to generate simulated outputs,\"Simulated harvest yield, dry matter\",\"Simulated above-ground biomass at harvest, dry matter\",Simulated anthesis date,Simulated maturity date,Simulated harvest date,\"Simulated leaf area index, maximum\",Total precipitation from planting to harvest,\"Simulated evapotranspiration, planting to harvest\",Simulated N uptake during season,Simulated N leached up to harvest maturity,\n!,text,number,text,text,text,text,code,number,code,number,code,code,text,number,text,text,decimal degrees,decimal degrees,text,text,text,yyyy-mm-dd,yyyy-mm-dd,kg/ha,kg/ha,yyyy-mm-dd,number,mm,text,number,kg[N]/ha,kg[P]/ha,kg[K]/ha,kg/ha,#,text,text,text,kg/ha,kg/ha,yyyy-mm-dd,yyyy-mm-dd,yyyy-mm-dd,m2/m2,mm,mm,kg/ha,kg/ha,\n#,EID,RUN#,FIELD_OVERLAY,SEASONAL_STRATEGY,EXNAME,TRT_NAME,CLIM_ID,CLIM_REP,REG_ID,STRATUM,RAP_ID,MAN_ID,INSTITUTION,ROTATION,WSTA_ID,SOIL_ID,FL_LAT,FL_LONG,CRID_text,CUL_ID,CUL_NAME,SDAT,PDATE,HWAH,CWAH,HDATE,IR#C,IR_TOT,IROP_text,FE_#,FEN_TOT,FEP_TOT,FEK_TOT,OM_TOT,TI_#,TIIMP_text,CROP_MODEL,MODEL_VER,HWAH_S,CWAH_S,ADAT_S,MDAT_S,HADAT_S,LAIX_S,PRCP_S,ETCP_S,NUCM_S,NLCM_S,\n";
    }

    protected static HashMap<String, String> extractEventData(HashMap<String, Object> dataset, String destModel) {
        destModel = destModel.toLowerCase();
        HashMap<String, String> results = new HashMap<String, String>();
        HashMap<String, Object> management = MapUtil.getRawBucket(dataset, "management");
        ArrayList<HashMap<String, String>> events = (ArrayList<HashMap<String, String>>) MapUtil.getObjectOr(management, "events", new ArrayList<HashMap<String, String>>());
        int irrCount = 0;
        int feCount = 0;
        int tilCount = 0;
        ArrayList<String> irop = new ArrayList<String>();
        ArrayList<String> timp = new ArrayList<String>();
        BigDecimal irrAmount, fenAmount, fekAmount, fepAmount, omAmount = null;
        try {
            irrAmount = new BigDecimal(0.0);
            fenAmount = new BigDecimal(0.0);
            fekAmount = new BigDecimal(0.0);
            fepAmount = new BigDecimal(0.0);
            omAmount  = new BigDecimal(0.0);
        } catch (Exception ex) {
            //Something really weird happened here. Like really really weird.
            log.error("Error converting 0.0 to a decimal, hard Java error");<commit_msg>Added climate ID support to ACMO_meta.data<commit_after>package org.agmip.acmo.util;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;

import org.agmip.util.MapUtil;
import org.agmip.ace.LookupCodes;
import org.agmip.ace.util.AcePathfinderUtil;
import org.agmip.dome.DomeUtil;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AcmoUtil {
    private static final Logger log = LoggerFactory.getLogger(AcmoUtil.class);
    /**
     * Do not instantiate
     */
    private AcmoUtil(){}


    /**
     * Write an ACMO file to the specified {@code outputPath} which includes
     * all the experiments in the {@code datapackage}.
     *
     * @param outputPath the directory to write the ACMO_meta.dat file.
     * @param datapackage a standard AgMIP package
     *
     */
    public static void writeAcmo(String outputPath, HashMap<String, Object> datapackage, String destModel) {
        HashMap<String, String> wstClimMap = new HashMap<String, String>();
        try {
            // Make sure the outputPath exists
            File f = new File(outputPath);
            f.mkdirs();
            String fileName   = outputPath+File.separator+"ACMO_meta.dat";
            log.debug("Attempting to write {}", fileName);
            FileWriter fw     = new FileWriter(fileName);
            BufferedWriter bw = new BufferedWriter(fw);
            // Index the Weather Stations
            for (HashMap<String, Object> wst : MapUtil.getRawPackageContents(datapackage, "weathers")) {
                wstClimMap.put(MapUtil.getValueOr(wst, "wst_id", ""), MapUtil.getValueOr(wst, "clim_id", "0XXX"));
            }

            try {
                // First write the header
                bw.write(generateAcmoHeader());
                 // Then write the lines
                ArrayList<HashMap<String, Object>> experiments = MapUtil.getRawPackageContents(datapackage, "experiments");
                for (HashMap<String, Object> experiment : experiments) {
                    // get WSTID and pass the CLIM_ID from that.
                    String wstId = MapUtil.getValueOr(experiment, "wst_id", "");
                    String climId = wstClimMap.get(wstId);
                    if (climId == null) {
                        climId = "0XXX";
                    }
                    String acmoData = extractAcmoData(experiment, destModel, climId);
                    log.debug("ACMO dataline: {}", acmoData);
                    bw.write(acmoData);
                    bw.write("\n");
                }
            } finally {
                // Finally close the file.
                log.debug("Attempting to close the ACMO file");
                bw.flush();
                bw.close();
                fw.close();
            }
        } catch (Exception ex) {
            log.error("Error writing ACMO_meta.dat: {}", ex.getMessage());
        }
    }

    /**
     * Extract a CSV-compatable ACMO line from a single experiment
     *
     * @param dataset a single AgMIP dataset
     *
     * @return ACMO compatible CSV line.
     */
    public static String extractAcmoData(HashMap<String, Object> dataset, String destModel, String climId) {
        ArrayList<String> acmoData = new ArrayList<String>();
        HashMap<String, Object> observed = MapUtil.getRawBucket(dataset, "observed");
        HashMap<String, String> events   = extractEventData(dataset, destModel);


        /**
         * Data to get:
         * wst_id (root)
         * soil_id (root)
         * fl_lat (root)
         * fl_lon (root)
         * crid (events)
         * cul_id (events)
         * cul_name (events)
         * sdat (root)
         * pdate (events)
         * hdate (observed)
         * hwah (observed)
         * cwah (observed)
         * ir_count (events)
         * ir_tot (events)
         * irop (events)
         * fe_count (events)
         * fen_tot (events)
         * fep_tot (events)
         * fek_tot (events)
         * omamt (events)
         * ti_count (events)
         * tiimp (events)
         * crop model (passed into writeAcmo())
         */

        String fieldOverlayString = MapUtil.getValueOr(dataset, "field_overlay", "");
        String seasonalStrategyString = MapUtil.getValueOr(dataset, "seasonal_strategy", "");
        HashMap<String, String> domeBase = new HashMap<String, String>();

        acmoData.add("*"); // Just an indication mark, this line stands alone
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "exname", ""))); // Will be generated by the database
        acmoData.add("1");
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "field_overlay", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "seasonal_strategy", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "exname", "")));
        acmoData.add(quoteMe(MapUtil.getValueOr(dataset, "trt_name", "")));
        acmoData.add(quoteMe(climId));
        acmoData.add("1");
        if (! seasonalStrategyString.equals("")) {
            domeBase = DomeUtil.unpackDomeName(seasonalStrategyString);
        } else if (! fieldOverlayString.equals("")){
            domeBase = DomeUtil.unpackDomeName(fieldOverlayString);
        }
        acmoData.add(quoteMe(MapUtil.getValueOr(domeBase, "region", ""))); // Region
        acmoData.add(quoteMe(MapUtil.getValueOr(domeBase, "stratum", ""))); // Stratum
        acmoData.add(MapUtil.getValueOr(domeBase, "rap_id", "")); // RAP ID
        acmoData.add(MapUtil.getValueOr(domeBase, "man_id", "")); // MAN ID
        acmoData.add("AgMIP"); // Institution
        acmoData.add(MapUtil.getValueOr(dataset, "rotation", "0"));
        acmoData.add(MapUtil.getValueOr(dataset, "wst_id", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "soil_id", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "fl_lat", ""));
        acmoData.add(MapUtil.getValueOr(dataset, "fl_long", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "crid", "")));
        acmoData.add(MapUtil.getValueOr(events, "cul_id", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "cul_name", "")));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(dataset, "sdat", "")));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(events, "pdate", "")));
        acmoData.add(MapUtil.getValueOr(observed, "hwah", ""));
        acmoData.add(MapUtil.getValueOr(observed, "cwah", ""));
        acmoData.add(correctDateFormat(MapUtil.getValueOr(observed, "hdate", "")));
        acmoData.add(MapUtil.getValueOr(events, "ir_count", ""));
        acmoData.add(MapUtil.getValueOr(events, "ir_tot", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "irop", "")));
        acmoData.add(MapUtil.getValueOr(events, "fe_count", ""));
        acmoData.add(MapUtil.getValueOr(events, "fen_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "fep_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "fek_tot", ""));
        acmoData.add(MapUtil.getValueOr(events, "omamt", ""));
        acmoData.add(MapUtil.getValueOr(events, "ti_count", ""));
        acmoData.add(quoteMe(MapUtil.getValueOr(events, "tiimp", "")));
        acmoData.add(destModel.toUpperCase());
        return joinList(acmoData, ",");
    }

    /**
     * Generates the standard ACMO header (with all the variables defined)
     *
     * @return ACMO header
     */
    public static String generateAcmoHeader() {
        return "!,Experiment ID,,Field Overlay ID,Seasonal Strategy ID,\"Name of experiment, field test or survey\",Treatment Name,4-character Climate ID code,Climate replication number for multiple realizations of weather data (ask Alex),Region ID,Regional stratum identification number,RAP ID,\"Management regimen ID, for multiple management regimens per RAP\",Names of institutions involved in collection of field or survey data,\"Crop rotation indicator (=1 to indicate that this is a continuous, multi-year simulation, =0 for single year simulations)\",Weather station ID,Soil ID,Site Latitude,Site Longitude,Crop type (common name) ,Crop model-specific cultivar ID,Cultivar name,Start of simulation date,Planting date,\"Observed harvested yield, dry weight\",Observed total above-ground biomass at harvest,Observed harvest date,Total number of irrigation events,Total amount of irrigation,Type of irrigation application,Total number of fertilizer applications,Total N applied,Total P applied,Total K applied,Manure and applied oganic matter,Total number of tillage applications,\"Tillage type (hand, animal or mechanized)\",\"Short name of crop model used for simulations (e.g., DSSAT, APSIM, Aquacrop, STICS, etc.)\",Model name and version number of the crop model used to generate simulated outputs,\"Simulated harvest yield, dry matter\",\"Simulated above-ground biomass at harvest, dry matter\",Simulated anthesis date,Simulated maturity date,Simulated harvest date,\"Simulated leaf area index, maximum\",Total precipitation from planting to harvest,\"Simulated evapotranspiration, planting to harvest\",Simulated N uptake during season,Simulated N leached up to harvest maturity,\n!,text,number,text,text,text,text,code,number,code,number,code,code,text,number,text,text,decimal degrees,decimal degrees,text,text,text,yyyy-mm-dd,yyyy-mm-dd,kg/ha,kg/ha,yyyy-mm-dd,number,mm,text,number,kg[N]/ha,kg[P]/ha,kg[K]/ha,kg/ha,#,text,text,text,kg/ha,kg/ha,yyyy-mm-dd,yyyy-mm-dd,yyyy-mm-dd,m2/m2,mm,mm,kg/ha,kg/ha,\n#,EID,RUN#,FIELD_OVERLAY,SEASONAL_STRATEGY,EXNAME,TRT_NAME,CLIM_ID,CLIM_REP,REG_ID,STRATUM,RAP_ID,MAN_ID,INSTITUTION,ROTATION,WST_ID,SOIL_ID,FL_LAT,FL_LONG,CRID_text,CUL_ID,CUL_NAME,SDAT,PDATE,HWAH,CWAH,HDATE,IR#C,IR_TOT,IROP_text,FE_#,FEN_TOT,FEP_TOT,FEK_TOT,OM_TOT,TI_#,TIIMP_text,CROP_MODEL,MODEL_VER,HWAH_S,CWAH_S,ADAT_S,MDAT_S,HADAT_S,LAIX_S,PRCP_S,ETCP_S,NUCM_S,NLCM_S,\n";
    }

    protected static HashMap<String, String> extractEventData(HashMap<String, Object> dataset, String destModel) {
        destModel = destModel.toLowerCase();
        HashMap<String, String> results = new HashMap<String, String>();
        HashMap<String, Object> management = MapUtil.getRawBucket(dataset, "management");
        ArrayList<HashMap<String, String>> events = (ArrayList<HashMap<String, String>>) MapUtil.getObjectOr(management, "events", new ArrayList<HashMap<String, String>>());
        int irrCount = 0;
        int feCount = 0;
        int tilCount = 0;
        ArrayList<String> irop = new ArrayList<String>();
        ArrayList<String> timp = new ArrayList<String>();
        BigDecimal irrAmount, fenAmount, fekAmount, fepAmount, omAmount = null;
        try {
            irrAmount = new BigDecimal(0.0);
            fenAmount = new BigDecimal(0.0);
            fekAmount = new BigDecimal(0.0);
            fepAmount = new BigDecimal(0.0);
            omAmount  = new BigDecimal(0.0);
        } catch (Exception ex) {
            //Something really weird happened here. Like really really weird.
            log.error("Error converting 0.0 to a decimal, hard Java error");
{'fraction_non_alphanumeric': 0.1283595922150139, 'fraction_numerical': 0.0024559777571825764, 'mean_word_length': 3.1422264875239923, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '246579', 'n_tokens_mistral': 6733, 'n_tokens_neox': 6568, 'n_words': 1400}