<commit_before>// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// https://github.com/Talend/data-prep/blob/master/LICENSE
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================

package org.talend.dataprep.schema.xls.serialization;

import static org.apache.commons.lang.StringUtils.EMPTY;
import static org.slf4j.LoggerFactory.getLogger;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.slf4j.Logger;
import org.talend.dataprep.api.dataset.ColumnMetadata;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.schema.xls.XlsSerializer;
import org.talend.dataprep.schema.xls.streaming.StreamingReader;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;

/**
 * Serialize XLSX file using stream parsing.
 */
public class XlsxStreamRunnable
    implements Runnable {

    /** This class' logger. */
    private static final Logger LOG = getLogger(XlsxStreamRunnable.class);

    /** Where to serialize the json. */
    private final OutputStream jsonOutput;

    /** The xlsx raw input. */
    private final InputStream rawContent;

    /** The dataset metadata to serialize. */
    private final DataSetMetadata metadata;

    /** The jackson factory to use for the serialization. */
    private final JsonFactory jsonFactory;

    /**
     * Constructor.
     *
     * @param jsonOutput Where to serialize the json.
     * @param rawContent The xlsx raw input.
     * @param metadata The dataset metadata to serialize.
     * @param factory The jackson factory to use for the serialization.
     */
    public XlsxStreamRunnable( OutputStream jsonOutput, InputStream rawContent, DataSetMetadata metadata, JsonFactory factory) {
        this.jsonOutput = jsonOutput;
        this.rawContent = rawContent;
        this.metadata = metadata;
        this.jsonFactory = factory;
    }

    /**
     * @see Runnable#run()
     */
    @Override
    public void run() {
        try {
            JsonGenerator generator = jsonFactory.createGenerator(jsonOutput);

            Workbook workbook = StreamingReader.builder() //
                .bufferSize(4096) //
                .rowCacheSize( 1 ) //
                .open(rawContent);

            Sheet sheet = StringUtils.isEmpty( metadata.getSheetName() ) ? //
                workbook.getSheetAt( 0 ) : workbook.getSheet( metadata.getSheetName() );


            generator.writeStartArray();

            for ( Row row : sheet){

                if (!XlsSerializer.isHeaderLine(row.getRowNum(), metadata.getRowMetadata().getColumns())) {
                    
                    generator.writeStartObject();
                    for (Cell cell : row) {
                            ColumnMetadata columnMetadata = metadata.getRowMetadata().getColumns().get(cell.getColumnIndex());
                            String cellValue = cell.getStringCellValue();
                            generator.writeFieldName(columnMetadata.getId());
                            if (cellValue != null) {
                                generator.writeString(cellValue);
                            } else {
                                generator.writeNull();
                            }

                    }
                    generator.writeEndObject();
                }

                generator.flush();
            }


            generator.writeEndArray();

            generator.flush();

        } catch (Exception e) {
            // Consumer may very well interrupt consumption of stream (in case of limit(n) use for sampling).
            // This is not an issue as consumer is allowed to partially consumes results, it's up to the
            // consumer to ensure data it consumed is consistent.
            LOG.debug("Unable to continue serialization for {}. Skipping remaining content.", metadata.getId(), e);
        } finally {
            try {
                jsonOutput.close();
            } catch (IOException e) {
                LOG.error("Unable to close output", e);
            }
        }
    }


    /**
     * Return a list of empty string from the given size.
     * @param size the wanted list size.
     * @return a list of empty string from the given size.
     */<commit_msg>[TDP-2024] data analytic does not like to not have empty columns so fill in with empty fields<commit_after>// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// https://github.com/Talend/data-prep/blob/master/LICENSE
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================

package org.talend.dataprep.schema.xls.serialization;

import static org.apache.commons.lang.StringUtils.EMPTY;
import static org.slf4j.LoggerFactory.getLogger;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.slf4j.Logger;
import org.talend.dataprep.api.dataset.ColumnMetadata;
import org.talend.dataprep.api.dataset.DataSetMetadata;
import org.talend.dataprep.schema.xls.XlsSerializer;
import org.talend.dataprep.schema.xls.streaming.StreamingReader;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;

/**
 * Serialize XLSX file using stream parsing.
 */
public class XlsxStreamRunnable
    implements Runnable {

    /** This class' logger. */
    private static final Logger LOG = getLogger(XlsxStreamRunnable.class);

    /** Where to serialize the json. */
    private final OutputStream jsonOutput;

    /** The xlsx raw input. */
    private final InputStream rawContent;

    /** The dataset metadata to serialize. */
    private final DataSetMetadata metadata;

    /** The jackson factory to use for the serialization. */
    private final JsonFactory jsonFactory;

    /**
     * Constructor.
     *
     * @param jsonOutput Where to serialize the json.
     * @param rawContent The xlsx raw input.
     * @param metadata The dataset metadata to serialize.
     * @param factory The jackson factory to use for the serialization.
     */
    public XlsxStreamRunnable( OutputStream jsonOutput, InputStream rawContent, DataSetMetadata metadata, JsonFactory factory) {
        this.jsonOutput = jsonOutput;
        this.rawContent = rawContent;
        this.metadata = metadata;
        this.jsonFactory = factory;
    }

    /**
     * @see Runnable#run()
     */
    @Override
    public void run() {
        try {
            JsonGenerator generator = jsonFactory.createGenerator(jsonOutput);

            Workbook workbook = StreamingReader.builder() //
                .bufferSize(4096) //
                .rowCacheSize( 1 ) //
                .open(rawContent);

            Sheet sheet = StringUtils.isEmpty( metadata.getSheetName() ) ? //
                workbook.getSheetAt( 0 ) : workbook.getSheet( metadata.getSheetName() );


            generator.writeStartArray();

            for ( Row row : sheet){
                if (!XlsSerializer.isHeaderLine(row.getRowNum(), metadata.getRowMetadata().getColumns())) {
                    
                    generator.writeStartObject();
                    // data quality Analyzer doesn't like to not have all columms even if we don't have any values
                    // so create so field with empty value otherwise we get exceptions
                    int i = 0;
                    for (ColumnMetadata columnMetadata : metadata.getRowMetadata().getColumns()){
                        Cell cell = row.getCell( i );
                        String cellValue = cell == null ? null : cell.getStringCellValue(); //StringUtils.EMPTY
                        generator.writeFieldName(columnMetadata.getId());
                        if (cellValue != null) {
                            generator.writeString(cellValue);
                        } else {
                            generator.writeNull();
                        }
                        i++;
                    }
                    generator.writeEndObject();
                }
            }


            generator.writeEndArray();

            generator.flush();

        } catch (Exception e) {
            // Consumer may very well interrupt consumption of stream (in case of limit(n) use for sampling).
            // This is not an issue as consumer is allowed to partially consumes results, it's up to the
            // consumer to ensure data it consumed is consistent.
            LOG.debug("Unable to continue serialization for {}. Skipping remaining content.", metadata.getId(), e);
        } finally {
            try {
                jsonOutput.close();
            } catch (IOException e) {
                LOG.error("Unable to close output", e);
            }
        }
    }


    /**
     * Return a list of empty string from the given size.
     * @param size the wanted list size.
     * @return a list of empty string from the given size.
     */
{'fraction_non_alphanumeric': 0.1222454880845548, 'fraction_numerical': 0.004885831089839466, 'mean_word_length': 2.1861499364675985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3408961', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2488, 'n_words': 724}