<commit_before>package chartreview

import gov.va.vinci.chartreview.Utils
import gov.va.vinci.chartreview.model.Project
import gov.va.vinci.chartreview.model.schema.AnnotationSchema
import gov.va.vinci.chartreview.model.schema.AnnotationSchemaRecord
import gov.va.vinci.chartreview.model.schema.AnnotationSchemaRecordDAO
import gov.va.vinci.chartreview.model.schema.AttributeDef
import gov.va.vinci.chartreview.model.schema.AttributeDefOptionDef
import gov.va.vinci.chartreview.model.schema.ClassDef
import gov.va.vinci.chartreview.model.schema.ClassRelDef
import gov.va.vinci.leo.descriptors.LeoTypeSystemDescription
import gov.va.vinci.leo.descriptors.TypeDescriptionBuilder
import grails.converters.XML
import groovy.util.slurpersupport.GPathResult
import org.apache.commons.dbutils.DbUtils

import java.sql.Connection
import java.sql.Timestamp
import java.text.SimpleDateFormat

/**
 * Service for dealing with annotation schemas.
 */
class AnnotationSchemaService {
    def projectService;

    /**
     * Copy an annotation schema. This creates new UUIDs.
     * @param p   the project that holds both the original schema and will hold the new copy.
     * @param record   the schema record to copy.
     * @param newName    the new name for the copied schema.
     * @param createdBy  the user that is creating the copy.
     * @return the new record.
     */
    public AnnotationSchemaRecord copy(Project p, AnnotationSchemaRecord record, String newName, String createdBy) {
        AnnotationSchemaRecord newRecord = new AnnotationSchemaRecord();
        AnnotationSchema schema = parseSchemaXml(record.serializationData, true);
        schema.name = newName;
        newRecord.name = schema.name;
        newRecord.description = schema.description;
        newRecord.id = schema.id;

        List<AnnotationSchema> schemas = new ArrayList<AnnotationSchema>();
        schema.applySorts();
        schemas.add(schema);
        XML xmlData= schemas as XML;
        newRecord.serializationData = xmlData.toString();
        newRecord.createdDate = new Date();
        newRecord.createdBy = createdBy;
        newRecord.lastModifiedBy = createdBy;
        newRecord.lastModifiedDate = newRecord.createdDate;
        newRecord.serializationVersion = record.version;
        newRecord.version = new Timestamp(System.currentTimeMillis());
        insert(p, newRecord);
        return newRecord;
    }

    /**
     * Insert a new annotation schema record into the project database.
     * @param p   the project to insert the schema record into.
     * @param record  the schema record to insert.
     */
    public void insert(Project p, AnnotationSchemaRecord record) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            dao.insert(record);
            if (!c.autoCommit) {
                c.commit();
            }
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Update a schema record.
     * @param p   the project the existing record resides in.
     * @param record  the record to update.
     */
    public void update(Project p, AnnotationSchemaRecord record) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            dao.update(record);
            if (!c.autoCommit) {
                c.commit();
            }
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Find a schema by name
     * @param p     The project to get the schema from.
     * @param name  The name of the annotation schema.
     * @return      The annotation schema, or null if not found.
     */
    public AnnotationSchemaRecord findByName(Project p, String name) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.findByName(name);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Get a schema by id
     * @param p     The project to get the schema from.
     * @param id    The id of the annotation schema.
     * @return      The annotation schema, or null if not found.
     */
    public AnnotationSchemaRecord get(Project p, String id) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.get(id);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Get all schemas in a given project
     * @param p     The project to get the schemas from.
     * @return      The annotation schemas, or an empty list if there are none.
     */
    public List<AnnotationSchemaRecord> getAll(Project p) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.getAll();
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * delete a schema in a given project
     * @param p     The project to get the schemas from.
     * @param id    The id of the schema to delete.
     * @return      The number of rows affected.
     */
    public Long delete(Project p, String id) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.delete(id);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Parse an xml representation of a schema into Java POJOS.
     * @param xml the xml to parse.
     * @param changeUUIDs  if true, all GUIDs in the xml schema are changed in the POJOS. This is useful
     * if copying a schema. If false, GUIDs are left as is.
     * @return the Object representation of the xml.
     *
     */
    public AnnotationSchema parseSchemaXml(String xml, boolean changeUUIDs = true) {
        Map idMap = null;
        def annotationSchemas = new XmlSlurper().parseText(xml).annotationSchema;
        AnnotationSchema annotationSchema = null;
        def it = annotationSchemas.iterator().next();

        if (changeUUIDs) {
            idMap = Utils.getReplacementsForTempIds(it)
        } else {
            // Get ID Map that is a straight map, no replacements.
            def ids = it.'**'.grep{ it.@id != '' }.'@id'*.text().collect{(String)it} as Set
            idMap= new HashMap<String, String>();
            ids.each { guid -> idMap.put(guid, guid) }
        }
        String id = Utils.safeGet(idMap, it?.@id?.text())
        String schemaName = it?.name?.text();
        int type = 0;
        String description = it?.description?.text();
        def tAnnotationSchema = new AnnotationSchema(id, schemaName, type, description);
        def SortedSet<AttributeDef> attributeDefs = this.prepareAttributeDefs(it?.attributeDefs?.attributeDef, idMap);
        def SortedSet<ClassDef> classDefs = this.prepareClassDefs(it?.classDefs?.classDef, tAnnotationSchema, attributeDefs, idMap);
        def SortedSet<ClassRelDef> classRelDefs = this.prepareClassRelDefs(it?.classRelDefs?.classRelDef, classDefs, attributeDefs, idMap);

        attributeDefs.each { it2 ->
            tAnnotationSchema.addAttributeDef(it2);
        }
        classDefs.each { it2 ->
            tAnnotationSchema.addClassDef(it2);
        }
        classRelDefs.each { it2 ->
            tAnnotationSchema.addClassRelDef(it2);
        }
        return tAnnotationSchema;
    }




    /**
     * Convert AttributeDefs elements in Xml into set of domain objects
     * @param nodes
     * @return set
     */
    protected SortedSet<AttributeDef> prepareAttributeDefs(GPathResult nodes, Map idMap){
        log.info('prepareAttributeDefs')
        def attributeDefSet = new TreeSet<AttributeDef>();

        nodes?.each { attrDf ->
            def attributeDef = new AttributeDef()<commit_msg>schema WIP - Fixed date format issue in service.<commit_after>package chartreview

import gov.va.vinci.chartreview.Utils
import gov.va.vinci.chartreview.model.Project
import gov.va.vinci.chartreview.model.schema.AnnotationSchema
import gov.va.vinci.chartreview.model.schema.AnnotationSchemaRecord
import gov.va.vinci.chartreview.model.schema.AnnotationSchemaRecordDAO
import gov.va.vinci.chartreview.model.schema.AttributeDef
import gov.va.vinci.chartreview.model.schema.AttributeDefOptionDef
import gov.va.vinci.chartreview.model.schema.ClassDef
import gov.va.vinci.chartreview.model.schema.ClassRelDef
import gov.va.vinci.leo.descriptors.LeoTypeSystemDescription
import gov.va.vinci.leo.descriptors.TypeDescriptionBuilder
import grails.converters.XML
import groovy.util.slurpersupport.GPathResult
import org.apache.commons.dbutils.DbUtils

import java.sql.Connection
import java.sql.Timestamp
import java.text.SimpleDateFormat

/**
 * Service for dealing with annotation schemas.
 */
class AnnotationSchemaService {
    def projectService;

    /**
     * Copy an annotation schema. This creates new UUIDs.
     * @param p   the project that holds both the original schema and will hold the new copy.
     * @param record   the schema record to copy.
     * @param newName    the new name for the copied schema.
     * @param createdBy  the user that is creating the copy.
     * @return the new record.
     */
    public AnnotationSchemaRecord copy(Project p, AnnotationSchemaRecord record, String newName, String createdBy) {
        AnnotationSchemaRecord newRecord = new AnnotationSchemaRecord();
        AnnotationSchema schema = parseSchemaXml(record.serializationData, true);
        schema.name = newName;
        newRecord.name = schema.name;
        newRecord.description = schema.description;
        newRecord.id = schema.id;

        List<AnnotationSchema> schemas = new ArrayList<AnnotationSchema>();
        schema.applySorts();
        schemas.add(schema);
        XML xmlData= schemas as XML;
        newRecord.serializationData = xmlData.toString();
        newRecord.createdDate = new Date();
        newRecord.createdBy = createdBy;
        newRecord.lastModifiedBy = createdBy;
        newRecord.lastModifiedDate = newRecord.createdDate;
        newRecord.serializationVersion = record.version;
        newRecord.version = new Timestamp(System.currentTimeMillis());
        insert(p, newRecord);
        return newRecord;
    }

    /**
     * Insert a new annotation schema record into the project database.
     * @param p   the project to insert the schema record into.
     * @param record  the schema record to insert.
     */
    public void insert(Project p, AnnotationSchemaRecord record) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            dao.insert(record);
            if (!c.autoCommit) {
                c.commit();
            }
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Update a schema record.
     * @param p   the project the existing record resides in.
     * @param record  the record to update.
     */
    public void update(Project p, AnnotationSchemaRecord record) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            dao.update(record);
            if (!c.autoCommit) {
                c.commit();
            }
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Find a schema by name
     * @param p     The project to get the schema from.
     * @param name  The name of the annotation schema.
     * @return      The annotation schema, or null if not found.
     */
    public AnnotationSchemaRecord findByName(Project p, String name) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.findByName(name);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Get a schema by id
     * @param p     The project to get the schema from.
     * @param id    The id of the annotation schema.
     * @return      The annotation schema, or null if not found.
     */
    public AnnotationSchemaRecord get(Project p, String id) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.get(id);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Get all schemas in a given project
     * @param p     The project to get the schemas from.
     * @return      The annotation schemas, or an empty list if there are none.
     */
    public List<AnnotationSchemaRecord> getAll(Project p) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.getAll();
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * delete a schema in a given project
     * @param p     The project to get the schemas from.
     * @param id    The id of the schema to delete.
     * @return      The number of rows affected.
     */
    public Long delete(Project p, String id) {
        Connection c = null;
        try {
            c = projectService.getDatabaseConnection(p);
            AnnotationSchemaRecordDAO dao = new AnnotationSchemaRecordDAO(c, Utils.getSQLTemplate(p.jdbcDriver));
            return dao.delete(id);
        } finally {
            if (c!= null) {
                DbUtils.close(c);
            }
        }
    }

    /**
     * Parse an xml representation of a schema into Java POJOS.
     * @param xml the xml to parse.
     * @param changeUUIDs  if true, all GUIDs in the xml schema are changed in the POJOS. This is useful
     * if copying a schema. If false, GUIDs are left as is.
     * @return the Object representation of the xml.
     *
     */
    public AnnotationSchema parseSchemaXml(String xml, boolean changeUUIDs = true) {
        Map idMap = null;
        def annotationSchemas = new XmlSlurper().parseText(xml).annotationSchema;
        AnnotationSchema annotationSchema = null;
        def it = annotationSchemas.iterator().next();

        if (changeUUIDs) {
            idMap = Utils.getReplacementsForTempIds(it)
        } else {
            // Get ID Map that is a straight map, no replacements.
            def ids = it.'**'.grep{ it.@id != '' }.'@id'*.text().collect{(String)it} as Set
            idMap= new HashMap<String, String>();
            ids.each { guid -> idMap.put(guid, guid) }
        }
        String id = Utils.safeGet(idMap, it?.@id?.text())
        String schemaName = it?.name?.text();
        int type = 0;
        String description = it?.description?.text();
        def tAnnotationSchema = new AnnotationSchema(id, schemaName, type, description);
        def SortedSet<AttributeDef> attributeDefs = this.prepareAttributeDefs(it?.attributeDefs?.attributeDef, idMap);
        def SortedSet<ClassDef> classDefs = this.prepareClassDefs(it?.classDefs?.classDef, tAnnotationSchema, attributeDefs, idMap);
        def SortedSet<ClassRelDef> classRelDefs = this.prepareClassRelDefs(it?.classRelDefs?.classRelDef, classDefs, attributeDefs, idMap);

        attributeDefs.each { it2 ->
            tAnnotationSchema.addAttributeDef(it2);
        }
        classDefs.each { it2 ->
            tAnnotationSchema.addClassDef(it2);
        }
        classRelDefs.each { it2 ->
            tAnnotationSchema.addClassRelDef(it2);
        }
        return tAnnotationSchema;
    }

    /**
     * Convert AttributeDefs elements in Xml into set of domain objects
     * @param nodes
     * @return set
     */
    protected SortedSet<AttributeDef> prepareAttributeDefs(GPathResult nodes, Map idMap){
        log.info('prepareAttributeDefs')
        def attributeDefSet = new TreeSet<AttributeDef>();

        nodes?.each { attrDf ->
            def attributeDef = new AttributeDef()
{'fraction_non_alphanumeric': 0.09013858664528691, 'fraction_numerical': 0.0008017409231474057, 'mean_word_length': 2.275131282820705, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2651784', 'n_tokens_mistral': 4880, 'n_tokens_neox': 4797, 'n_words': 1310}