<commit_before>package com.realcomp.data.record.writer;

import com.realcomp.data.Operation;
import com.realcomp.data.conversion.ConversionException;
import com.realcomp.data.conversion.Converter;
import com.realcomp.data.conversion.MultiFieldConverter;
import com.realcomp.data.record.AfterLastRecord;
import com.realcomp.data.record.BeforeFirstRecord;
import com.realcomp.data.record.Record;
import com.realcomp.data.schema.FileSchema;
import com.realcomp.data.schema.SchemaException;
import com.realcomp.data.schema.SchemaField;
import com.realcomp.data.validation.Severity;
import com.realcomp.data.validation.ValidationException;
import com.realcomp.data.validation.Validator;
import com.realcomp.data.view.RecordView;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;

/**
 *
 * @author krenfro
 */
public abstract class BaseFileWriter implements RecordWriter{

    protected static final Logger log = Logger.getLogger(BaseFileWriter.class.getName());

    protected OutputStream out;
    protected FileSchema schema;
    protected Severity validationExceptionThreshold = DEFAULT_VALIDATION_THREASHOLD;
    protected long count;
    protected boolean beforeFirstOperationsRun = false;
    
    public BaseFileWriter(){
    }
    
    public BaseFileWriter(BaseFileWriter copy){
        validationExceptionThreshold = copy.validationExceptionThreshold;
        try{
            schema = new FileSchema(copy.schema);
        }
        catch(SchemaException ex){
            throw new IllegalStateException(ex); //should never happen
        }
    }
    
    @Override
    public Severity getValidationExceptionThreshold() {
        return validationExceptionThreshold;
    }

    @Override
    public void setValidationExceptionThreshold(Severity severity) {
        this.validationExceptionThreshold = severity;
    }
    
    @Override
    public void open(OutputStream out){
        if (out == null)
            throw new IllegalArgumentException("out is null");
        this.out = out;
        count = 0;        
    }

    @Override
    public void close(){

        try {
            executeAfterLastOperations();
        }
        catch (ValidationException ex) {
            log.log(Level.WARNING, null, ex);
        }
        catch (ConversionException ex) {
            log.log(Level.WARNING, null, ex);
        }

        IOUtils.closeQuietly(out);
        beforeFirstOperationsRun = false;
    }

    protected void executeAfterLastOperations() throws ValidationException, ConversionException{
        
        if (schema != null){
            List<Operation> afterLast = schema.getAfterLastOperations();
            if (afterLast != null){
                for (Operation op: afterLast){
                    operate(op, "" + this.getCount(), new AfterLastRecord());
                }
            }
        }
    }

    protected void executeBeforeFirstOperations() throws ValidationException, ConversionException{

        if (schema != null){
            List<Operation> beforeFirst = schema.getBeforeFirstOperations();
            if (beforeFirst != null){
                for (Operation op: beforeFirst){
                    operate(op, "", new BeforeFirstRecord());
                }
            }
        }
    }
    
    @Override
    public void setSchema(FileSchema schema) throws SchemaException{
        this.schema = schema;
    }

    @Override
    public FileSchema getSchema(){
        return schema;
    }

    @Override
    public void write(RecordView recordView)
        throws IOException, ValidationException, ConversionException{

        write(recordView.getRecord());
    }
    
    @Override
    public void write(Record record)
            throws IOException, ValidationException, ConversionException{

        if (schema == null)
            throw new IllegalStateException("schema not specified");
        if (record == null)
            throw new IllegalArgumentException("record is null");

        if (!beforeFirstOperationsRun){
            executeBeforeFirstOperations();
            beforeFirstOperationsRun = true;
        }

        write(record, schema.getFields());
        count++;
    }

 
    protected void write(Record record, List<SchemaField> fields)
            throws ValidationException, ConversionException, IOException{

        for (SchemaField field: fields)
            write(record, field);
    }


    protected abstract void write(Record record, SchemaField field)
       throws ValidationException, ConversionException, IOException;


    protected String toString(Record record, SchemaField schemaField)
            throws ValidationException, ConversionException{

        Object value = record.get(schemaField.getName());
        String data = "";
        if (value != null)
            data = value.toString();
        
        try{
            data = operate(schema.getBeforeOperations(), data, record);
            data = operate(schemaField.getOperations(), data, record);
            data = operate(schema.getAfterOperations(), data, record);
        }
        catch(ValidationException ex){
            throw new ValidationException(schemaField.getName() + " " + ex.getMessage(), ex);
        }
        catch(ConversionException ex){
            throw new ConversionException(schemaField.getName() + " " + ex.getMessage(), ex);
        }

        return data;
    }

    protected String operate(List<Operation> operations, String data, Record record)
            throws ConversionException, ValidationException{
        
        if (operations == null)<commit_msg>Attempt at adding alias support to FileWriter<commit_after>package com.realcomp.data.record.writer;

import com.realcomp.data.Operation;
import com.realcomp.data.conversion.Alias;
import com.realcomp.data.conversion.ConversionException;
import com.realcomp.data.conversion.Converter;
import com.realcomp.data.conversion.MultiFieldConverter;
import com.realcomp.data.record.AfterLastRecord;
import com.realcomp.data.record.BeforeFirstRecord;
import com.realcomp.data.record.Record;
import com.realcomp.data.schema.FileSchema;
import com.realcomp.data.schema.SchemaException;
import com.realcomp.data.schema.SchemaField;
import com.realcomp.data.validation.Severity;
import com.realcomp.data.validation.ValidationException;
import com.realcomp.data.validation.Validator;
import com.realcomp.data.view.RecordView;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.io.IOUtils;

/**
 *
 * @author krenfro
 */
public abstract class BaseFileWriter implements RecordWriter{

    protected static final Logger log = Logger.getLogger(BaseFileWriter.class.getName());

    protected OutputStream out;
    protected FileSchema schema;
    protected Severity validationExceptionThreshold = DEFAULT_VALIDATION_THREASHOLD;
    protected long count;
    protected boolean beforeFirstOperationsRun = false;
    
    public BaseFileWriter(){
    }
    
    public BaseFileWriter(BaseFileWriter copy){
        validationExceptionThreshold = copy.validationExceptionThreshold;
        try{
            schema = new FileSchema(copy.schema);
        }
        catch(SchemaException ex){
            throw new IllegalStateException(ex); //should never happen
        }
    }
    
    @Override
    public Severity getValidationExceptionThreshold() {
        return validationExceptionThreshold;
    }

    @Override
    public void setValidationExceptionThreshold(Severity severity) {
        this.validationExceptionThreshold = severity;
    }
    
    @Override
    public void open(OutputStream out){
        if (out == null)
            throw new IllegalArgumentException("out is null");
        this.out = out;
        count = 0;        
    }

    @Override
    public void close(){

        try {
            executeAfterLastOperations();
        }
        catch (ValidationException ex) {
            log.log(Level.WARNING, null, ex);
        }
        catch (ConversionException ex) {
            log.log(Level.WARNING, null, ex);
        }

        IOUtils.closeQuietly(out);
        beforeFirstOperationsRun = false;
    }

    protected void executeAfterLastOperations() throws ValidationException, ConversionException{
        
        if (schema != null){
            List<Operation> afterLast = schema.getAfterLastOperations();
            if (afterLast != null){
                for (Operation op: afterLast){
                    operate(op, "" + this.getCount(), new AfterLastRecord());
                }
            }
        }
    }

    protected void executeBeforeFirstOperations() throws ValidationException, ConversionException{

        if (schema != null){
            List<Operation> beforeFirst = schema.getBeforeFirstOperations();
            if (beforeFirst != null){
                for (Operation op: beforeFirst){
                    operate(op, "", new BeforeFirstRecord());
                }
            }
        }
    }
    
    @Override
    public void setSchema(FileSchema schema) throws SchemaException{
        this.schema = schema;
    }

    @Override
    public FileSchema getSchema(){
        return schema;
    }

    @Override
    public void write(RecordView recordView)
        throws IOException, ValidationException, ConversionException{

        write(recordView.getRecord());
    }
    
    @Override
    public void write(Record record)
            throws IOException, ValidationException, ConversionException{

        if (schema == null)
            throw new IllegalStateException("schema not specified");
        if (record == null)
            throw new IllegalArgumentException("record is null");

        if (!beforeFirstOperationsRun){
            executeBeforeFirstOperations();
            beforeFirstOperationsRun = true;
        }

        write(record, schema.getFields());
        count++;
    }

 
    protected void write(Record record, List<SchemaField> fields)
            throws ValidationException, ConversionException, IOException{

        for (SchemaField field: fields)
            write(record, field);
    }


    protected abstract void write(Record record, SchemaField field)
       throws ValidationException, ConversionException, IOException;


    protected String toString(Record record, SchemaField schemaField)
            throws ValidationException, ConversionException{

        Object value = record.get(schemaField.getName());
        String data = "";
        if (value == null){
            //try aliases
            List<Operation> operations = schemaField.getOperations();
            if (operations != null){
                for (Operation op: operations){
                    if (op instanceof Alias){
                        value = record.get(((Alias) op).getName());
                        if (value != null){
                            data = value.toString();
                            break;
                        }
                    }
                }
            }
        }
        else{
            data = value.toString();
        }
        
        try{
            data = operate(schema.getBeforeOperations(), data, record);
            data = operate(schemaField.getOperations(), data, record);
            data = operate(schema.getAfterOperations(), data, record);
        }
        catch(ValidationException ex){
            throw new ValidationException(schemaField.getName() + " " + ex.getMessage(), ex);
        }
        catch(ConversionException ex){
            throw new ConversionException(schemaField.getName() + " " + ex.getMessage(), ex);
        }

        return data;
    }

    protected String operate(List<Operation> operations, String data, Record record)
            throws ConversionException, ValidationException{
        
        if (operations == null)
{'fraction_non_alphanumeric': 0.08843423799582463, 'fraction_numerical': 0.00016701461377870565, 'mean_word_length': 2.4374282433983927, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6000045', 'n_tokens_mistral': 2934, 'n_tokens_neox': 2872, 'n_words': 703}