<commit_before>package name.abuchen.portfolio.datatransfer.csv;

import java.math.BigDecimal;
import java.text.MessageFormat;
import java.text.ParseException;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import name.abuchen.portfolio.Messages;
import name.abuchen.portfolio.datatransfer.Extractor;
import name.abuchen.portfolio.datatransfer.SecurityCache;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Column;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Field;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.FieldFormat;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Header;
import name.abuchen.portfolio.model.Client;
import name.abuchen.portfolio.model.Portfolio;
import name.abuchen.portfolio.model.PortfolioTransaction;
import name.abuchen.portfolio.model.Security;
import name.abuchen.portfolio.model.Transaction;
import name.abuchen.portfolio.money.CurrencyConverter;
import name.abuchen.portfolio.money.CurrencyConverterImpl;
import name.abuchen.portfolio.money.ExchangeRateProviderFactory;
import name.abuchen.portfolio.money.Money;
import name.abuchen.portfolio.money.Values;
import name.abuchen.portfolio.snapshot.PortfolioSnapshot;
import name.abuchen.portfolio.snapshot.SecurityPosition;
import name.abuchen.portfolio.util.Iban;
import name.abuchen.portfolio.util.Isin;

public abstract class CSVExtractor implements Extractor
{
    public abstract List<Field> getFields();

    public abstract List<Item> extract(int skipLines, List<String[]> rawValues, Map<String, Column> field2column,
                    List<Exception> errors);

    @Override
    public List<Item> extract(SecurityCache securityCache, Extractor.InputFile file, List<Exception> errors)
    {
        throw new UnsupportedOperationException();
    }

    public int getDefaultSkipLines()
    {
        return 0;
    }

    public Header.Type getDefaultHeadering()
    {
        return Header.Type.FIRST;
    }

    public String getDefaultEncoding()
    {
        return "UTF-8"; //$NON-NLS-1$
    }

    public <E extends Enum<E>> EnumMap<E, String> getDefaultEnum(Class<E> enumType)
    {
        return null;
    }

    public String[] getDefaultHeader()
    {
        return null;
    }

    protected String getText(String name, String[] rawValues, Map<String, Column> field2column)
    {
        Column column = field2column.get(name);
        if (column == null)
            return null;

        int columnIndex = column.getColumnIndex();

        if (columnIndex < 0 || columnIndex >= rawValues.length)
            return null;

        String value = rawValues[columnIndex];
        return value != null && value.trim().length() == 0 ? null : value;
    }

    protected String getPattern(String name, String[] rawValues, Map<String, Column> field2column, String pattern)
    {
        Column column = field2column.get(name);
        if (column == null)
            return null;

        int columnIndex = column.getColumnIndex();

        if (columnIndex < 0 || columnIndex >= rawValues.length)
            return null;

        String value = rawValues[columnIndex];
        if (value == null)
            return null;

        value = value.trim().toUpperCase();

        Pattern Ipattern = Pattern.compile("\\b(" + pattern + ")\\b"); //$NON-NLS-1$ //$NON-NLS-2$
        Matcher Imatcher = Ipattern.matcher(value);
        if (Imatcher.find())
            value = Imatcher.group(1);

        return value.length() == 0 ? null : value;
    }

    protected String getIBAN(String name, String[] rawValues, Map<String, Column> field2column)
    {
        return getPattern(name, rawValues, field2column, Iban.PATTERN);
    }

    protected String getISIN(String name, String[] rawValues, Map<String, Column> field2column)
    {
        return getPattern(name, rawValues, field2column, Isin.PATTERN);
    }

    protected Long getAmount(String name, String[] rawValues, Map<String, Column> field2column) throws ParseException
    {
        return getValue(name, rawValues, field2column, Values.Amount);
    }

    protected Long getQuote(String name, String[] rawValues, Map<String, Column> field2column) throws ParseException
    {
        return getValue(name, rawValues, field2column, Values.Quote);
    }

    protected Long getValue(String name, String[] rawValues, Map<String, Column> field2column, Values<Long> values)
                    throws ParseException
    {
        String value = getText(name, rawValues, field2column);
        if (value == null)
            return null;
        Number num;
        try
        {
            num = (Number) field2column.get(name).getFormat().getFormat().parseObject(value);
        }
        catch (ParseException | UnsupportedOperationException | IllegalArgumentException e)
        {
            if (e instanceof ParseException)
            {
                throw new ParseException(e.getMessage() + ": " + MessageFormat.format(Messages.CSVImportGenericColumnLabel, field2column.get(name).getColumnIndex()), new Exception().getStackTrace()[0].getLineNumber()); //$NON-NLS-1$
            }
            else
                throw e;
        }
        return Long.valueOf((long) Math.round(num.doubleValue() * values.factor()));
    }

    protected LocalDateTime getDate(String dateColumn, String timeColumn, String[] rawValues,
                    Map<String, Column> field2column) throws ParseException
    {
        String dateValue = getText(dateColumn, rawValues, field2column);
        if (dateValue == null)
            return null;
        Date date;
        try
        {
            date = (Date) field2column.get(dateColumn).getFormat().getFormat().parseObject(dateValue);
        }
        catch (ParseException | UnsupportedOperationException | IllegalArgumentException e)
        {
            if (e instanceof ParseException)
                throw new ParseException(e.getMessage() + ": " + MessageFormat.format(Messages.CSVImportGenericColumnLabel, field2column.get(dateColumn).getColumnIndex()), new Exception().getStackTrace()[0].getLineNumber()); //$NON-NLS-1$
            else
                throw e;
        }
        LocalDateTime result = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault());

        if (timeColumn == null)
            return result;

        String timeValue = getText(timeColumn, rawValues, field2column);
        if (timeValue != null)
        {
            int p = timeValue.indexOf(':');
            if (p > 0)
            {
                try
                {
                    int hour = Integer.parseInt(timeValue.substring(0, p));
                    int minute = Integer.parseInt(timeValue.substring(p + 1));

                    result = result.withHour(hour).withMinute(minute);
                }
                catch (NumberFormatException | DateTimeException ignore)
                {
                    // ignore time, just use the date - not parseable<commit_msg>fixed time import in case time is given including seconds<commit_after>package name.abuchen.portfolio.datatransfer.csv;

import java.math.BigDecimal;
import java.text.MessageFormat;
import java.text.ParseException;
import java.time.DateTimeException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Date;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import name.abuchen.portfolio.Messages;
import name.abuchen.portfolio.datatransfer.Extractor;
import name.abuchen.portfolio.datatransfer.SecurityCache;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Column;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Field;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.FieldFormat;
import name.abuchen.portfolio.datatransfer.csv.CSVImporter.Header;
import name.abuchen.portfolio.model.Client;
import name.abuchen.portfolio.model.Portfolio;
import name.abuchen.portfolio.model.PortfolioTransaction;
import name.abuchen.portfolio.model.Security;
import name.abuchen.portfolio.model.Transaction;
import name.abuchen.portfolio.money.CurrencyConverter;
import name.abuchen.portfolio.money.CurrencyConverterImpl;
import name.abuchen.portfolio.money.ExchangeRateProviderFactory;
import name.abuchen.portfolio.money.Money;
import name.abuchen.portfolio.money.Values;
import name.abuchen.portfolio.snapshot.PortfolioSnapshot;
import name.abuchen.portfolio.snapshot.SecurityPosition;
import name.abuchen.portfolio.util.Iban;
import name.abuchen.portfolio.util.Isin;

public abstract class CSVExtractor implements Extractor
{
    public abstract List<Field> getFields();

    public abstract List<Item> extract(int skipLines, List<String[]> rawValues, Map<String, Column> field2column,
                    List<Exception> errors);

    @Override
    public List<Item> extract(SecurityCache securityCache, Extractor.InputFile file, List<Exception> errors)
    {
        throw new UnsupportedOperationException();
    }

    public int getDefaultSkipLines()
    {
        return 0;
    }

    public Header.Type getDefaultHeadering()
    {
        return Header.Type.FIRST;
    }

    public String getDefaultEncoding()
    {
        return "UTF-8"; //$NON-NLS-1$
    }

    public <E extends Enum<E>> EnumMap<E, String> getDefaultEnum(Class<E> enumType)
    {
        return null;
    }

    public String[] getDefaultHeader()
    {
        return null;
    }

    protected String getText(String name, String[] rawValues, Map<String, Column> field2column)
    {
        Column column = field2column.get(name);
        if (column == null)
            return null;

        int columnIndex = column.getColumnIndex();

        if (columnIndex < 0 || columnIndex >= rawValues.length)
            return null;

        String value = rawValues[columnIndex];
        return value != null && value.trim().length() == 0 ? null : value;
    }

    protected String getPattern(String name, String[] rawValues, Map<String, Column> field2column, String pattern)
    {
        Column column = field2column.get(name);
        if (column == null)
            return null;

        int columnIndex = column.getColumnIndex();

        if (columnIndex < 0 || columnIndex >= rawValues.length)
            return null;

        String value = rawValues[columnIndex];
        if (value == null)
            return null;

        value = value.trim().toUpperCase();

        Pattern Ipattern = Pattern.compile("\\b(" + pattern + ")\\b"); //$NON-NLS-1$ //$NON-NLS-2$
        Matcher Imatcher = Ipattern.matcher(value);
        if (Imatcher.find())
            value = Imatcher.group(1);

        return value.length() == 0 ? null : value;
    }

    protected String getIBAN(String name, String[] rawValues, Map<String, Column> field2column)
    {
        return getPattern(name, rawValues, field2column, Iban.PATTERN);
    }

    protected String getISIN(String name, String[] rawValues, Map<String, Column> field2column)
    {
        return getPattern(name, rawValues, field2column, Isin.PATTERN);
    }

    protected Long getAmount(String name, String[] rawValues, Map<String, Column> field2column) throws ParseException
    {
        return getValue(name, rawValues, field2column, Values.Amount);
    }

    protected Long getQuote(String name, String[] rawValues, Map<String, Column> field2column) throws ParseException
    {
        return getValue(name, rawValues, field2column, Values.Quote);
    }

    protected Long getValue(String name, String[] rawValues, Map<String, Column> field2column, Values<Long> values)
                    throws ParseException
    {
        String value = getText(name, rawValues, field2column);
        if (value == null)
            return null;
        Number num;
        try
        {
            num = (Number) field2column.get(name).getFormat().getFormat().parseObject(value);
        }
        catch (ParseException | UnsupportedOperationException | IllegalArgumentException e)
        {
            if (e instanceof ParseException)
            {
                throw new ParseException(e.getMessage() + ": " + MessageFormat.format(Messages.CSVImportGenericColumnLabel, field2column.get(name).getColumnIndex()), new Exception().getStackTrace()[0].getLineNumber()); //$NON-NLS-1$
            }
            else
                throw e;
        }
        return Long.valueOf((long) Math.round(num.doubleValue() * values.factor()));
    }

    protected LocalDateTime getDate(String dateColumn, String timeColumn, String[] rawValues,
                    Map<String, Column> field2column) throws ParseException
    {
        String dateValue = getText(dateColumn, rawValues, field2column);
        if (dateValue == null)
            return null;
        Date date;
        try
        {
            date = (Date) field2column.get(dateColumn).getFormat().getFormat().parseObject(dateValue);
        }
        catch (ParseException | UnsupportedOperationException | IllegalArgumentException e)
        {
            if (e instanceof ParseException)
                throw new ParseException(e.getMessage() + ": " + MessageFormat.format(Messages.CSVImportGenericColumnLabel, field2column.get(dateColumn).getColumnIndex()), new Exception().getStackTrace()[0].getLineNumber()); //$NON-NLS-1$
            else
                throw e;
        }
        LocalDateTime result = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault());

        if (timeColumn == null)
            return result;

        String timeValue = getText(timeColumn, rawValues, field2column);
        if (timeValue != null)
        {
            String[] timeToks = timeValue.split(":"); //$NON-NLS-1$
            if (timeToks.length > 1)
            {
                try
                {
                    int hour = Integer.parseInt(timeToks[0]);
                    int minute = Integer.parseInt(timeToks[1]);

                    result = result.withHour(hour).withMinute(minute);
                }
                catch (NumberFormatException | DateTimeException ignore)
                {
                    // ignore time, just use the date - not parseable
{'fraction_non_alphanumeric': 0.10495242064318955, 'fraction_numerical': 0.005487254289088005, 'mean_word_length': 2.7999472156241754, 'pattern_counts': {'":': 5, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5835532', 'n_tokens_mistral': 4077, 'n_tokens_neox': 3955, 'n_words': 883}