<commit_before>package name.abuchen.portfolio.online.impl;

import static name.abuchen.portfolio.online.impl.YahooHelper.FMT_PRICE;
import static name.abuchen.portfolio.online.impl.YahooHelper.FMT_QUOTE_DATE;
import static name.abuchen.portfolio.online.impl.YahooHelper.asDate;
import static name.abuchen.portfolio.online.impl.YahooHelper.asNumber;
import static name.abuchen.portfolio.online.impl.YahooHelper.asPrice;
import static name.abuchen.portfolio.online.impl.YahooHelper.stripQuotes;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.MessageFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
import java.util.stream.Collectors;

import name.abuchen.portfolio.Messages;
import name.abuchen.portfolio.model.Exchange;
import name.abuchen.portfolio.model.LatestSecurityPrice;
import name.abuchen.portfolio.model.Security;
import name.abuchen.portfolio.model.SecurityPrice;
import name.abuchen.portfolio.online.QuoteFeed;
import name.abuchen.portfolio.util.Dates;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;

public class YahooFinanceQuoteFeed implements QuoteFeed
{
    public static final String ID = "YAHOO"; //$NON-NLS-1$

    private static final String LATEST_URL = "http://finance.yahoo.com/d/quotes.csv?s={0}&f=sl1d1hgpv"; //$NON-NLS-1$
    // s = symbol
    // l1 = last trade (price only)
    // d1 = last trade date
    // h = day's high
    // g = day's low
    // p = previous close
    // v = volume
    // Source = http://cliffngan.net/a/13

    private static final String SEARCH_URL = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query={0}&callback=YAHOO.Finance.SymbolSuggest.ssCallback"; //$NON-NLS-1$

    @SuppressWarnings("nls")
    private static final String HISTORICAL_URL = "http://ichart.finance.yahoo.com/table.csv?ignore=.csv" //
                    + "&s={0}" // ticker symbol
                    + "&a={1}&b={2}&c={3}" // begin
                    + "&d={4}&e={5}&f={6}" // end
                    + "&g=d"; // daily

    @Override
    public String getId()
    {
        return ID;
    }

    @Override
    public String getName()
    {
        return Messages.LabelYahooFinance;
    }

    @Override
    public final boolean updateLatestQuotes(List<Security> securities, List<Exception> errors)
    {
        Map<String, List<Security>> symbol2security = securities.stream()
                        //
                        .filter(s -> s.getTickerSymbol() != null)
                        .collect(Collectors.groupingBy(s -> s.getTickerSymbol().toUpperCase(Locale.ROOT)));

        String symbolString = symbol2security.keySet().stream().collect(Collectors.joining("+")); //$NON-NLS-1$

        boolean isUpdated = false;

        String url = MessageFormat.format(LATEST_URL, symbolString.toString());

        String line = null;
        try (BufferedReader reader = openReader(url, errors))
        {
            if (reader == null)
                return false;

            while ((line = reader.readLine()) != null)
            {
                String[] values = line.split(","); //$NON-NLS-1$
                if (values.length != 7)
                {
                    errors.add(new IOException(MessageFormat.format(Messages.MsgUnexpectedValue, line)));
                    return false;
                }

                String symbol = stripQuotes(values[0]);
                List<Security> forSymbol = symbol2security.remove(symbol);
                if (forSymbol == null)
                {
                    errors.add(new IOException(MessageFormat.format(Messages.MsgUnexpectedSymbol, symbol, line)));
                    continue;
                }

                long lastTrade = asPrice(values[1]);

                Date lastTradeDate = asDate(values[2]);
                if (lastTradeDate == null) // can't work w/o date
                    lastTradeDate = Dates.today();

                long daysHigh = asPrice(values[3]);

                long daysLow = asPrice(values[4]);

                long previousClose = asPrice(values[5]);

                int volume = asNumber(values[6]);

                LatestSecurityPrice price = new LatestSecurityPrice(lastTradeDate, lastTrade);
                price.setHigh(daysHigh);
                price.setLow(daysLow);
                price.setPreviousClose(previousClose);
                price.setVolume(volume);

                for (Security security : forSymbol)
                {
                    boolean isAdded = security.setLatest(price);
                    isUpdated = isUpdated || isAdded;
                }
            }

            for (String symbol : symbol2security.keySet())
                errors.add(new IOException(MessageFormat.format(Messages.MsgMissingResponse, symbol)));
        }
        catch (NumberFormatException | ParseException e)
        {
            errors.add(new IOException(MessageFormat.format(Messages.MsgErrorsConvertingValue, line), e));
        }
        catch (IOException e)
        {
            errors.add(e);
        }

        return isUpdated;
    }

    @Override
    public final boolean updateHistoricalQuotes(Security security, List<Exception> errors)
    {
        Calendar start = caculateStart(security);

        List<SecurityPrice> quotes = internalGetQuotes(SecurityPrice.class, security, start.getTime(), errors);

        boolean isUpdated = false;
        if (quotes != null)
        {
            for (SecurityPrice p : quotes)
            {
                boolean isAdded = security.addPrice(p);
                isUpdated = isUpdated || isAdded;<commit_msg>Fixed Yahoo quote update to not fail all updates on one parsing error<commit_after>package name.abuchen.portfolio.online.impl;

import static name.abuchen.portfolio.online.impl.YahooHelper.FMT_PRICE;
import static name.abuchen.portfolio.online.impl.YahooHelper.FMT_QUOTE_DATE;
import static name.abuchen.portfolio.online.impl.YahooHelper.asDate;
import static name.abuchen.portfolio.online.impl.YahooHelper.asNumber;
import static name.abuchen.portfolio.online.impl.YahooHelper.asPrice;
import static name.abuchen.portfolio.online.impl.YahooHelper.stripQuotes;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.MessageFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
import java.util.stream.Collectors;

import name.abuchen.portfolio.Messages;
import name.abuchen.portfolio.model.Exchange;
import name.abuchen.portfolio.model.LatestSecurityPrice;
import name.abuchen.portfolio.model.Security;
import name.abuchen.portfolio.model.SecurityPrice;
import name.abuchen.portfolio.online.QuoteFeed;
import name.abuchen.portfolio.util.Dates;

import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;

public class YahooFinanceQuoteFeed implements QuoteFeed
{
    public static final String ID = "YAHOO"; //$NON-NLS-1$

    private static final String LATEST_URL = "http://finance.yahoo.com/d/quotes.csv?s={0}&f=sl1d1hgpv"; //$NON-NLS-1$
    // s = symbol
    // l1 = last trade (price only)
    // d1 = last trade date
    // h = day's high
    // g = day's low
    // p = previous close
    // v = volume
    // Source = http://cliffngan.net/a/13

    private static final String SEARCH_URL = "http://d.yimg.com/autoc.finance.yahoo.com/autoc?query={0}&callback=YAHOO.Finance.SymbolSuggest.ssCallback"; //$NON-NLS-1$

    @SuppressWarnings("nls")
    private static final String HISTORICAL_URL = "http://ichart.finance.yahoo.com/table.csv?ignore=.csv" //
                    + "&s={0}" // ticker symbol
                    + "&a={1}&b={2}&c={3}" // begin
                    + "&d={4}&e={5}&f={6}" // end
                    + "&g=d"; // daily

    @Override
    public String getId()
    {
        return ID;
    }

    @Override
    public String getName()
    {
        return Messages.LabelYahooFinance;
    }

    @Override
    public final boolean updateLatestQuotes(List<Security> securities, List<Exception> errors)
    {
        Map<String, List<Security>> symbol2security = securities.stream()
                        //
                        .filter(s -> s.getTickerSymbol() != null)
                        .collect(Collectors.groupingBy(s -> s.getTickerSymbol().toUpperCase(Locale.ROOT)));

        String symbolString = symbol2security.keySet().stream().collect(Collectors.joining("+")); //$NON-NLS-1$

        boolean isUpdated = false;

        String url = MessageFormat.format(LATEST_URL, symbolString.toString());

        try (BufferedReader reader = openReader(url, errors))
        {
            if (reader == null)
                return false;

            String line = null;
            while ((line = reader.readLine()) != null)
            {
                String[] values = line.split(","); //$NON-NLS-1$
                if (values.length != 7)
                {
                    errors.add(new IOException(MessageFormat.format(Messages.MsgUnexpectedValue, line)));
                    return false;
                }

                String symbol = stripQuotes(values[0]);
                List<Security> forSymbol = symbol2security.remove(symbol);
                if (forSymbol == null)
                {
                    errors.add(new IOException(MessageFormat.format(Messages.MsgUnexpectedSymbol, symbol, line)));
                    continue;
                }

                try
                {
                    LatestSecurityPrice price = buildPrice(values);

                    for (Security security : forSymbol)
                    {
                        boolean isAdded = security.setLatest(price);
                        isUpdated = isUpdated || isAdded;
                    }
                }
                catch (NumberFormatException | ParseException e)
                {
                    errors.add(new IOException(MessageFormat.format(Messages.MsgErrorsConvertingValue, line), e));
                }
            }

            for (String symbol : symbol2security.keySet())
                errors.add(new IOException(MessageFormat.format(Messages.MsgMissingResponse, symbol)));
        }
        catch (IOException e)
        {
            errors.add(e);
        }

        return isUpdated;
    }

    private LatestSecurityPrice buildPrice(String[] values) throws ParseException
    {
        long lastTrade = asPrice(values[1]);

        Date lastTradeDate = asDate(values[2]);
        if (lastTradeDate == null) // can't work w/o date
            lastTradeDate = Dates.today();

        long daysHigh = asPrice(values[3]);

        long daysLow = asPrice(values[4]);

        long previousClose = asPrice(values[5]);

        int volume = asNumber(values[6]);

        LatestSecurityPrice price = new LatestSecurityPrice(lastTradeDate, lastTrade);
        price.setHigh(daysHigh);
        price.setLow(daysLow);
        price.setPreviousClose(previousClose);
        price.setVolume(volume);

        return price;
    }

    @Override
    public final boolean updateHistoricalQuotes(Security security, List<Exception> errors)
    {
        Calendar start = caculateStart(security);

        List<SecurityPrice> quotes = internalGetQuotes(SecurityPrice.class, security, start.getTime(), errors);

        boolean isUpdated = false;
        if (quotes != null)
        {
            for (SecurityPrice p : quotes)
            {
                boolean isAdded = security.addPrice(p);
                isUpdated = isUpdated || isAdded;
{'fraction_non_alphanumeric': 0.11260819859545974, 'fraction_numerical': 0.005226196308998857, 'mean_word_length': 2.283378016085791, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6038928', 'n_tokens_mistral': 3572, 'n_tokens_neox': 3401, 'n_words': 677}