<commit_before>package nl.mpi.kinnate.gedcomimport;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import javax.swing.JProgressBar;
import javax.swing.JTextArea;
import nl.mpi.arbil.userstorage.SessionStorage;
import nl.mpi.arbil.util.BugCatcherManager;
import nl.mpi.kinnate.kindocument.EntityDocument;
import nl.mpi.kinnate.kindocument.ImportTranslator;
import nl.mpi.kinnate.uniqueidentifiers.UniqueIdentifier;

/**
 * Document : EntityImporter
 * Created on : May 30, 2011, 10:28:59 AM
 * Author : <NAME>
 */
public class EntityImporter implements GenericImporter {

    protected JProgressBar progressBar = null;
    protected URI inputFileUri = null; // used only for copying resource files
    protected JTextArea importTextArea;
    private int inputLineCount;
    private int currntLineCounter;
    protected String inputFileMd5Sum;
    protected boolean overwriteExisting;
    protected HashMap<String, HashSet<UniqueIdentifier>> createdNodeIds;
    HashMap<String, EntityDocument> createdDocuments = new HashMap<String, EntityDocument>();
//    private MetadataBuilder metadataBuilder;
    private SessionStorage sessionStorage;

    public EntityImporter(JProgressBar progressBarLocal, JTextArea importTextAreaLocal, boolean overwriteExistingLocal, SessionStorage sessionStorage) {
        overwriteExisting = overwriteExistingLocal;
        importTextArea = importTextAreaLocal;
        progressBar = progressBarLocal;
//        metadataBuilder = new MetadataBuilder();
        createdNodeIds = new HashMap<String, HashSet<UniqueIdentifier>>();
        this.sessionStorage = sessionStorage;
    }

    public HashMap<String, HashSet<UniqueIdentifier>> getCreatedNodeIds() {
        return createdNodeIds;
    }

    public void appendToTaskOutput(String lineOfText) {
        importTextArea.append(lineOfText + "\n");
        importTextArea.setCaretPosition(importTextArea.getText().length());
    }

    public void calculateFileNameAndFileLength(InputStream inputStream) {
        // count the lines in the file (for progress) and calculate the md5 sum (for unique file naming)
        currntLineCounter = 0;
        try {
            MessageDigest digest = MessageDigest.getInstance("MD5");

            byte[] buffer = new byte[1024];
            int numRead;
            do {
                numRead = inputStream.read(buffer);
                if (numRead > 0) {
                    digest.update(buffer, 0, numRead);
                }
            } while (numRead != -1);
            inputStream.close();

            StringBuilder hexString = new StringBuilder();
            byte[] md5sum = digest.digest();
            for (int byteCounter = 0; byteCounter < md5sum.length; ++byteCounter) {
                hexString.append(Integer.toHexString(0x0100 + (md5sum[byteCounter] & 0x00FF)).substring(1));
            }
            inputFileMd5Sum = hexString.toString();
        } catch (NoSuchAlgorithmException algorithmException) {
            BugCatcherManager.getBugCatcher().logError(algorithmException);
        } catch (IOException iOException) {
            BugCatcherManager.getBugCatcher().logError(iOException);
        }
    }
<commit_msg>Corrected an issue where the progress bar did not show the increments when importing sample gedcom data. refs #1866<commit_after>package nl.mpi.kinnate.gedcomimport;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import javax.swing.JProgressBar;
import javax.swing.JTextArea;
import nl.mpi.arbil.userstorage.SessionStorage;
import nl.mpi.arbil.util.BugCatcherManager;
import nl.mpi.kinnate.kindocument.EntityDocument;
import nl.mpi.kinnate.kindocument.ImportTranslator;
import nl.mpi.kinnate.uniqueidentifiers.UniqueIdentifier;

/**
 * Document : EntityImporter
 * Created on : May 30, 2011, 10:28:59 AM
 * Author : <NAME>
 */
public class EntityImporter implements GenericImporter {

    protected JProgressBar progressBar = null;
    protected URI inputFileUri = null; // used only for copying resource files
    protected JTextArea importTextArea;
    private int inputLineCount;
    private int currntLineCounter;
    protected String inputFileMd5Sum;
    protected boolean overwriteExisting;
    protected HashMap<String, HashSet<UniqueIdentifier>> createdNodeIds;
    HashMap<String, EntityDocument> createdDocuments = new HashMap<String, EntityDocument>();
//    private MetadataBuilder metadataBuilder;
    private SessionStorage sessionStorage;

    public EntityImporter(JProgressBar progressBarLocal, JTextArea importTextAreaLocal, boolean overwriteExistingLocal, SessionStorage sessionStorage) {
        overwriteExisting = overwriteExistingLocal;
        importTextArea = importTextAreaLocal;
        progressBar = progressBarLocal;
//        metadataBuilder = new MetadataBuilder();
        createdNodeIds = new HashMap<String, HashSet<UniqueIdentifier>>();
        this.sessionStorage = sessionStorage;
    }

    public HashMap<String, HashSet<UniqueIdentifier>> getCreatedNodeIds() {
        return createdNodeIds;
    }

    public void appendToTaskOutput(String lineOfText) {
        importTextArea.append(lineOfText + "\n");
        importTextArea.setCaretPosition(importTextArea.getText().length());
    }

    public void calculateFileNameAndFileLength(InputStream inputStream) {
        // count the lines in the file (for progress) and calculate the md5 sum (for unique file naming)
        currntLineCounter = 0;
        byte newLineByte = "\n".getBytes()[0]; // to keep things simple we only use the one char where, this will not work on old macs which are the only ones to still use only \r, however new macs now use \n so this is a minumal issue.
        try {
            MessageDigest digest = MessageDigest.getInstance("MD5");

            byte[] buffer = new byte[1024];
            int numRead;
            do {
                numRead = inputStream.read(buffer);
                if (numRead > 0) {
                    for (byte currentByte : buffer) {
                        if (currentByte == newLineByte) {
                            inputLineCount++;
                        }
                    }
                    digest.update(buffer, 0, numRead);
                }
            } while (numRead != -1);
            inputStream.close();

            StringBuilder hexString = new StringBuilder();
            byte[] md5sum = digest.digest();
            for (int byteCounter = 0; byteCounter < md5sum.length; ++byteCounter) {
                hexString.append(Integer.toHexString(0x0100 + (md5sum[byteCounter] & 0x00FF)).substring(1));
            }
            inputFileMd5Sum = hexString.toString();
        } catch (NoSuchAlgorithmException algorithmException) {
            BugCatcherManager.getBugCatcher().logError(algorithmException);
        } catch (IOException iOException) {
            BugCatcherManager.getBugCatcher().logError(iOException);
        }
    }

{'fraction_non_alphanumeric': 0.08607157110549421, 'fraction_numerical': 0.010509511773313822, 'mean_word_length': 3.1720310765815762, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '533923', 'n_tokens_mistral': 2059, 'n_tokens_neox': 1960, 'n_words': 491}