<commit_before>package com.realcomp.data.util;

import au.com.bytecode.opencsv.CSVParser;
import com.realcomp.data.record.Record;
import com.realcomp.data.schema.Field;
import com.realcomp.data.schema.FieldList;
import com.realcomp.data.schema.Schema;
import com.realcomp.data.schema.xml.XStreamFactory;
import com.thoughtworks.xstream.XStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;

/**
 * Schema generator for delimited files.
 * Uses the first record in a delimited file to generate a schema.
 *
 * @author krenfro
 */
public class SchemaGenerator{

    private static final Logger logger = Logger.getLogger(SchemaGenerator.class.getName());

    private String delimiter = "TAB";

    public void generate(InputStream in, OutputStream out) throws IOException{
        String header = new BufferedReader(new InputStreamReader(in)).readLine();
        String[] fieldNames = getFieldNames(header);
        Schema schema = buildSchema(fieldNames);
        writeSchema(schema, out);
    }

    public Schema generate(File file) throws IOException{
        String header = null;
        try (BufferedReader reader = new BufferedReader(new FileReader(file))){
            header = reader.readLine();
        }
        String[] fieldNames = getFieldNames(header);
        return buildSchema(fieldNames);
    }

    public Schema generate(Record record) throws IOException{
        String[] fieldNames = record.keySet().toArray(new String[record.keySet().size()]);
        return buildSchema(fieldNames);
    }


    protected String toXml(Schema schema){
        XStream xstream = XStreamFactory.build(true);
        StringWriter temp = new StringWriter();
        xstream.toXML(schema, temp);
        return temp.getBuffer().toString();
    }

    protected void writeSchema(Schema schema, OutputStream out) throws IOException{

        String xml = toXml(schema);
        xml = clean(xml);
        out.write(xml.getBytes());
    }

    protected String[] getFieldNames(String header) throws IOException{
        CSVParser parser;

        if (delimiter.equals("\t") || delimiter.equalsIgnoreCase("TAB")){
            parser = new CSVParser('\t', '\u0000');
        }
        else if (delimiter.equalsIgnoreCase("CSV")){
            parser = new CSVParser(',', CSVParser.DEFAULT_QUOTE_CHARACTER);
        }
        else{
            parser = new CSVParser(delimiter.charAt(0), CSVParser.DEFAULT_QUOTE_CHARACTER);
        }

        return parser.parseLine(header);
    }

    protected Schema buildSchema(String[] fieldNames){
        Schema schema = new Schema();
        Map<String, String> format = new HashMap<>();
        format.put("header", "true");

        if (delimiter.equals("\t") || delimiter.equalsIgnoreCase("TAB")){
            format.put("type", "TAB");
        }
        else if (delimiter.equalsIgnoreCase("CSV")){
            format.put("type", "CSV");
        }
        else{
            format.put("type", "" + delimiter.charAt(0));
        }

        schema.setFormat(format);


        int count = 1;
        FieldList fieldList = schema.getDefaultFieldList();
        for (String fieldName : fieldNames){
            fieldList.add(new Field(fieldName.isEmpty() ? "FIELD" + count : fieldName));
            count++;
        }

        return schema;
    }

    protected String clean(String dirty){
        String header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
        header = header.concat("<rc:schema\n");
        header = header.concat("   xmlns:rc=\"http://www.real-comp.com/realcomp-data/schema/file-schema/1.2\"\n");
        header = header.concat("   xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n");
        header =
                header.concat("   xsi:schemaLocation=\"http://www.real-comp.com/realcomp-data/schema/file-schema/1.2 http://www.real-comp.com/realcomp-data/schema/file-schema/1.2/file-schema.xsd\">\n");

        String clean = header.concat(dirty.replace("<schema>", ""));
        clean = clean.replace("</schema>", "</rc:schema>");
        clean = clean.replaceAll(" length=\"0\"", "");
        clean = clean.replaceAll(" classifier=\".*\"", "");
        clean = clean.concat("\n");
        return clean;
    }

    public String getDelimiter(){<commit_msg>fixed bug in SchemaGenerator.  This utility was counting on a Schema always having one FieldList, but this is no longer the case.<commit_after>package com.realcomp.data.util;

import au.com.bytecode.opencsv.CSVParser;
import com.realcomp.data.record.Record;
import com.realcomp.data.schema.Field;
import com.realcomp.data.schema.FieldList;
import com.realcomp.data.schema.Schema;
import com.realcomp.data.schema.xml.XStreamFactory;
import com.thoughtworks.xstream.XStream;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Logger;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;

/**
 * Schema generator for delimited files.
 * Uses the first record in a delimited file to generate a schema.
 *
 * @author krenfro
 */
public class SchemaGenerator{

    private static final Logger logger = Logger.getLogger(SchemaGenerator.class.getName());

    private String delimiter = "TAB";

    public void generate(InputStream in, OutputStream out) throws IOException{
        String header = new BufferedReader(new InputStreamReader(in)).readLine();
        String[] fieldNames = getFieldNames(header);
        Schema schema = buildSchema(fieldNames);
        writeSchema(schema, out);
    }

    public Schema generate(File file) throws IOException{
        String header = null;
        try (BufferedReader reader = new BufferedReader(new FileReader(file))){
            header = reader.readLine();
        }
        String[] fieldNames = getFieldNames(header);
        return buildSchema(fieldNames);
    }

    public Schema generate(Record record) throws IOException{
        String[] fieldNames = record.keySet().toArray(new String[record.keySet().size()]);
        return buildSchema(fieldNames);
    }


    protected String toXml(Schema schema){
        XStream xstream = XStreamFactory.build(true);
        StringWriter temp = new StringWriter();
        xstream.toXML(schema, temp);
        return temp.getBuffer().toString();
    }

    protected void writeSchema(Schema schema, OutputStream out) throws IOException{

        String xml = toXml(schema);
        xml = clean(xml);
        out.write(xml.getBytes());
    }

    protected String[] getFieldNames(String header) throws IOException{
        CSVParser parser;

        if (delimiter.equals("\t") || delimiter.equalsIgnoreCase("TAB")){
            parser = new CSVParser('\t', '\u0000');
        }
        else if (delimiter.equalsIgnoreCase("CSV")){
            parser = new CSVParser(',', CSVParser.DEFAULT_QUOTE_CHARACTER);
        }
        else{
            parser = new CSVParser(delimiter.charAt(0), CSVParser.DEFAULT_QUOTE_CHARACTER);
        }

        return parser.parseLine(header);
    }

    protected Schema buildSchema(String[] fieldNames){
        Schema schema = new Schema();
        Map<String, String> format = new HashMap<>();
        format.put("header", "true");

        if (delimiter.equals("\t") || delimiter.equalsIgnoreCase("TAB")){
            format.put("type", "TAB");
        }
        else if (delimiter.equalsIgnoreCase("CSV")){
            format.put("type", "CSV");
        }
        else{
            format.put("type", "" + delimiter.charAt(0));
        }

        schema.setFormat(format);


        int count = 1;
        FieldList fieldList = new FieldList();
        for (String fieldName : fieldNames){
            fieldList.add(new Field(fieldName.isEmpty() ? "FIELD" + count : fieldName));
            count++;
        }
        schema.addFieldList(fieldList);

        return schema;
    }

    protected String clean(String dirty){
        String header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
        header = header.concat("<rc:schema\n");
        header = header.concat("   xmlns:rc=\"http://www.real-comp.com/realcomp-data/schema/file-schema/1.2\"\n");
        header = header.concat("   xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n");
        header =
                header.concat("   xsi:schemaLocation=\"http://www.real-comp.com/realcomp-data/schema/file-schema/1.2 http://www.real-comp.com/realcomp-data/schema/file-schema/1.2/file-schema.xsd\">\n");

        String clean = header.concat(dirty.replace("<schema>", ""));
        clean = clean.replace("</schema>", "</rc:schema>");
        clean = clean.replaceAll(" length=\"0\"", "");
        clean = clean.replaceAll(" classifier=\".*\"", "");
        clean = clean.concat("\n");
        return clean;
    }

    public String getDelimiter(){
{'fraction_non_alphanumeric': 0.13384440070444423, 'fraction_numerical': 0.004558168445042992, 'mean_word_length': 3.000828843762951, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4889835', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2712, 'n_words': 594}