<commit_before>package com.ardoq;

import com.ardoq.model.*;
import com.ardoq.util.SyncUtil;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.xssf.usermodel.*;

import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;

public class ExcelImport {

    private static final String fieldColMapping_prefix = "fieldColMapping_";
    private static final String compMappingPrefix = "compMapping_";
    private static String componentSeparator = "::";
    private static String token;
    private static String host;

    private final static Properties config = new Properties();
    private static HashMap<String, String> columnMapping = new HashMap<String, String>();
    private static String modelName;
    private static String workspaceName;
    private static String componentSheet;
    private static String componentFile;
    private static String referenceFile;
    private static HashMap<String, String> compMapping = new HashMap<String, String>();


    private static String descriptionColumn;
    private static ArdoqClient client;
    private static SyncUtil ardoqSync;
    private static String organization;
    private static String referenceSheet;
    private static String referenceDefaultLinkType;
    private static int referenceLinkTypeColumn;
    private static int referenceStartFromRow;
    private static int referenceSourceColumn;
    private static int referenceStartFromColumn;

    static HashMap<String, Component> cachedMap = new HashMap<String, Component>();

    public static void main(String[] args) throws IOException {
        if (args.length > 0) {
            String configFile = args[0];
            System.out.println("Loading config: " + configFile);
            config.load(new FileReader(configFile));
            parseConfig();
            initClient();
            syncComponents();

            if (null != referenceFile)
            {
                syncReferences();
            }

            if (config.getProperty("deleteMissing", "no").trim().equals("YES")){
                ardoqSync.deleteNotSyncedItems();
            }

            System.out.println(ardoqSync.getReport());


        }
        else
        {
            System.err.println("Path to config file must be first argument.");
        }
    }

    private static void syncComponents() throws IOException {

        System.out.println("Loading Excel file: "+componentFile);
        XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(componentFile));

        System.out.println("Finding spread sheet: "+componentSheet);
        // Import components
        XSSFSheet compSheet = workbook.getSheet(componentSheet);
        System.out.println("Analyzing sheet");
        XSSFRow columnRow = compSheet.getRow(0);

        int descriptionIndex = -1;
        HashMap<Integer, String> compTypeMap = new HashMap<Integer,String>();
        HashMap<Integer,String> fieldTypeMap = new HashMap<Integer,String>();
        int componentCellRange = 0;
        Iterator<Cell> cellIterator = columnRow.cellIterator();
        while (cellIterator.hasNext()){
            Cell cell = cellIterator.next();
            String heading = cell.getStringCellValue().trim();

            if (heading.length() > 0){
            boolean found = false;
            if (heading.equals(descriptionColumn))
            {
                found = true;
                descriptionIndex = cell.getColumnIndex();
                System.out.println("Found description column: "+heading+", "+descriptionIndex);
            }
            if (compMapping.containsKey(heading)){
                found = true;
                System.out.println("Found componentType heading: "+heading+" , "+cell.getColumnIndex());
                compTypeMap.put(cell.getColumnIndex(), compMapping.get(heading));
                componentCellRange = (cell.getColumnIndex() > componentCellRange) ? cell.getColumnIndex() : componentCellRange;
                compMapping.remove(heading);
            }
            if (columnMapping.containsKey(heading))
            {
                found = true;
                System.out.println("Found field heading: "+heading+" , "+cell.getColumnIndex());
                fieldTypeMap.put(cell.getColumnIndex(),columnMapping.get(heading));
                columnMapping.remove(heading);
            }
                if (!found)
                {
                    System.out.println("Ignored column, no component or field mapping found: "+heading);
                }
            }
        }

        System.out.println(ardoqSync.getModel().getComponentTypes());

        for (String key : columnMapping.keySet()){
            System.out.println("WARNING: Couldn't find Column: "+key+" - cannot map it to field: "+columnMapping.get("key"));
        }

        for (String key : compMapping.keySet()){
            System.out.println("WARNING: Couldn't find Column: "+key+" - cannot map it to component type: "+compMapping.get("key"));
        }

        int rowIndex = 1;
        XSSFRow row = compSheet.getRow(rowIndex);

        while (row != null)
        {
           ExcelComponent currentComp = null;
           String parentPath = null;
           for (int i = 0; i < componentCellRange+1; i++){
               if (compTypeMap.containsKey(i) && row.getCell(i) != null && row.getCell(i).getRawValue().trim().length()> 1){
                   String name = (row.getCell(i).getCellType() != Cell.CELL_TYPE_STRING ) ? row.getCell(i).getRawValue().trim() : row.getCell(i).getStringCellValue();
                   parentPath = (parentPath == null) ? name : parentPath+"."+name;
                   Component c = getComponent(parentPath, name, compTypeMap.get(i));
                   currentComp = new ExcelComponent(parentPath, c, currentComp);
               }
           }
           if (currentComp != null) {
               System.out.println("Found component: "+currentComp.getMyComponent().getName());
               currentComp.getMyComponent().setDescription((descriptionIndex != -1 && row.getCell(descriptionIndex) != null) ? row.getCell(descriptionIndex).getStringCellValue() : "");
               HashMap<String, Object> fields = new HashMap<String, Object>();
               for (Integer column : fieldTypeMap.keySet()){
                   Cell fieldValue = row.getCell(column);
                   if (fieldValue != null) {
                       String key = fieldTypeMap.get(column);
                       if (fieldValue.getCellType() == Cell.CELL_TYPE_BOOLEAN)
                       {
                           fields.put(key, fieldValue.getBooleanCellValue());
                       }
                       else if (fieldValue.getCellType() == Cell.CELL_TYPE_NUMERIC){
                           fields.put(key, fieldValue.getNumericCellValue());
                       }
                       else if (fieldValue.getStringCellValue().trim().length() > 0){
                           fields.put(key, fieldValue.getStringCellValue().trim());
                       }
                   }
               }
               currentComp.getMyComponent().setFields(fields);
           }
           row = compSheet.getRow(++rowIndex);
        }
        for (ExcelComponent ec : ExcelComponent.getRootNodes())
        {
            ec.setMyComponent(ardoqSync.addComponent(ec.getMyComponent()));
            //Update cache.<commit_msg>Performance improvement, stop reading after 10 empty rows.<commit_after>package com.ardoq;

import com.ardoq.model.*;
import com.ardoq.util.SyncUtil;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.xssf.usermodel.*;

import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;

public class ExcelImport {

    private static final String fieldColMapping_prefix = "fieldColMapping_";
    private static final String compMappingPrefix = "compMapping_";
    private static final int CONSECUTIVE_EMPTY_ROWS_MAX = 10;
    private static String componentSeparator = "::";
    private static String token;
    private static String host;

    private final static Properties config = new Properties();
    private static HashMap<String, String> columnMapping = new HashMap<String, String>();
    private static String modelName;
    private static String workspaceName;
    private static String componentSheet;
    private static String componentFile;
    private static String referenceFile;
    private static HashMap<String, String> compMapping = new HashMap<String, String>();


    private static String descriptionColumn;
    private static ArdoqClient client;
    private static SyncUtil ardoqSync;
    private static String organization;
    private static String referenceSheet;
    private static String referenceDefaultLinkType;
    private static int referenceLinkTypeColumn;
    private static int referenceStartFromRow;
    private static int referenceSourceColumn;
    private static int referenceStartFromColumn;

    static HashMap<String, Component> cachedMap = new HashMap<String, Component>();

    public static void main(String[] args) throws IOException {
        if (args.length > 0) {
            String configFile = args[0];
            System.out.println("Loading config: " + configFile);
            config.load(new FileReader(configFile));
            parseConfig();
            initClient();
            syncComponents();

            if (null != referenceFile)
            {
                syncReferences();
            }

            if (config.getProperty("deleteMissing", "no").trim().equals("YES")){
                ardoqSync.deleteNotSyncedItems();
            }

            System.out.println(ardoqSync.getReport());


        }
        else
        {
            System.err.println("Path to config file must be first argument.");
        }
    }

    private static void syncComponents() throws IOException {

        System.out.println("Loading Excel file: "+componentFile);
        XSSFWorkbook workbook = new XSSFWorkbook(new FileInputStream(componentFile));

        System.out.println("Finding spread sheet: "+componentSheet);
        // Import components
        XSSFSheet compSheet = workbook.getSheet(componentSheet);
        System.out.println("Analyzing sheet");
        XSSFRow columnRow = compSheet.getRow(0);

        int descriptionIndex = -1;
        HashMap<Integer, String> compTypeMap = new HashMap<Integer,String>();
        HashMap<Integer,String> fieldTypeMap = new HashMap<Integer,String>();
        int componentCellRange = 0;
        Iterator<Cell> cellIterator = columnRow.cellIterator();
        while (cellIterator.hasNext()){
            Cell cell = cellIterator.next();
            String heading = cell.getStringCellValue().trim();

            if (heading.length() > 0){
            boolean found = false;
            if (heading.equals(descriptionColumn))
            {
                found = true;
                descriptionIndex = cell.getColumnIndex();
                System.out.println("Found description column: "+heading+", "+descriptionIndex);
            }
            if (compMapping.containsKey(heading)){
                found = true;
                System.out.println("Found componentType heading: "+heading+" , "+cell.getColumnIndex());
                compTypeMap.put(cell.getColumnIndex(), compMapping.get(heading));
                componentCellRange = (cell.getColumnIndex() > componentCellRange) ? cell.getColumnIndex() : componentCellRange;
                compMapping.remove(heading);
            }
            if (columnMapping.containsKey(heading))
            {
                found = true;
                System.out.println("Found field heading: "+heading+" , "+cell.getColumnIndex());
                fieldTypeMap.put(cell.getColumnIndex(),columnMapping.get(heading));
                columnMapping.remove(heading);
            }
                if (!found)
                {
                    System.out.println("Ignored column, no component or field mapping found: "+heading);
                }
            }
        }

        System.out.println(ardoqSync.getModel().getComponentTypes());

        for (String key : columnMapping.keySet()){
            System.out.println("WARNING: Couldn't find Column: "+key+" - cannot map it to field: "+columnMapping.get("key"));
        }

        for (String key : compMapping.keySet()){
            System.out.println("WARNING: Couldn't find Column: "+key+" - cannot map it to component type: "+compMapping.get("key"));
        }

        int rowIndex = 1;
        int emptyRowCount = 0;
        XSSFRow row = compSheet.getRow(rowIndex);

        while (row != null)
        {
           ExcelComponent currentComp = null;
           String parentPath = null;
           for (int i = 0; i < componentCellRange+1; i++){
               if (compTypeMap.containsKey(i) && row.getCell(i) != null && row.getCell(i).getRawValue().trim().length()> 1){
                   String name = (row.getCell(i).getCellType() != Cell.CELL_TYPE_STRING ) ? row.getCell(i).getRawValue().trim() : row.getCell(i).getStringCellValue();
                   parentPath = (parentPath == null) ? name : parentPath+"."+name;
                   Component c = getComponent(parentPath, name, compTypeMap.get(i));
                   currentComp = new ExcelComponent(parentPath, c, currentComp);
               }
           }
           if (currentComp != null) {
               System.out.println("Found component: "+currentComp.getMyComponent().getName());
               currentComp.getMyComponent().setDescription((descriptionIndex != -1 && row.getCell(descriptionIndex) != null) ? row.getCell(descriptionIndex).getStringCellValue() : "");
               HashMap<String, Object> fields = new HashMap<String, Object>();
               for (Integer column : fieldTypeMap.keySet()){
                   Cell fieldValue = row.getCell(column);
                   if (fieldValue != null) {
                       String key = fieldTypeMap.get(column);
                       if (fieldValue.getCellType() == Cell.CELL_TYPE_BOOLEAN)
                       {
                           fields.put(key, fieldValue.getBooleanCellValue());
                       }
                       else if (fieldValue.getCellType() == Cell.CELL_TYPE_NUMERIC){
                           fields.put(key, fieldValue.getNumericCellValue());
                       }
                       else if (fieldValue.getStringCellValue().trim().length() > 0){
                           fields.put(key, fieldValue.getStringCellValue().trim());
                       }
                   }
               }
               currentComp.getMyComponent().setFields(fields);
           }
           row = compSheet.getRow(++rowIndex);
           if (row != null) {
               if (row.cellIterator().hasNext()) {
                   emptyRowCount = 0;
               } else {
                   emptyRowCount++;
               }
               if (emptyRowCount == CONSECUTIVE_EMPTY_ROWS_MAX) {
                   row = null;
               }
           }
        }
        for (ExcelComponent ec : ExcelComponent.getRootNodes())
        {
            ec.setMyComponent(ardoqSync.addComponent(ec.getMyComponent()));
            //Update cache.
{'fraction_non_alphanumeric': 0.10807453416149068, 'fraction_numerical': 0.001961425302386401, 'mean_word_length': 2.162944582299421, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2480503', 'n_tokens_mistral': 3848, 'n_tokens_neox': 3699, 'n_words': 879}