<commit_before>package ee.tuleva.onboarding.comparisons.fundvalue.retrieval;

import com.rollbar.Rollbar;
import ee.tuleva.onboarding.comparisons.fundvalue.ComparisonFund;
import ee.tuleva.onboarding.comparisons.fundvalue.FundValue;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Slf4j
@Service
@RequiredArgsConstructor
public class WorldIndexValueRetriever implements FundValueRetriever {

    private final Environment environment;
    private final RestTemplate restTemplate;

    @Value("${logging.rollbar.accessToken:#{null}}")
    private String accessToken;

    private static final String SOURCE_URL = "https://docs.google.com/spreadsheets/d/125aXusxnf-Mij-4D4W5Qnneb8H4chjebnIMpyrAfRkI/gviz/tq?tqx=out:csv&gid=619370394";

    @Override
    public ComparisonFund getRetrievalFund() {
        return ComparisonFund.MARKET;
    }

    @Override
    public List<FundValue> retrieveValuesForRange(Instant startDate, Instant endDate) {
        return restTemplate.execute(SOURCE_URL, HttpMethod.GET, getRequestCallback(), getResponseExtractor(startDate, endDate));
    }

    private ResponseExtractor<List<FundValue>> getResponseExtractor(Instant startDate, Instant endDate) {
        return response -> {
            if (response.getStatusCode() == HttpStatus.OK) {
                InputStream body = response.getBody();
                BufferedReader reader = new BufferedReader(new InputStreamReader(body, StandardCharsets.UTF_8));
                Stream<String> lines = reader.lines().skip(3);
                return parseLines(lines, startDate, endDate);
            } else {
                warn("Failed to fetch World Index values");
                return Collections.emptyList();
            }
        };
    }

    private List<FundValue> parseLines(Stream<String> lines, Instant startDate, Instant endDate) {
        return lines
            .map(this::parseLine)
            .filter(Optional::isPresent)
            .map(Optional::get)
            .filter((FundValue value) -> {
                Instant time = value.getTime();
                return (startDate.isBefore(time) || startDate.equals(time)) && (endDate.isAfter(time) || endDate.equals(time));
            })
            .collect(Collectors.toList());
    }

    private Optional<FundValue> parseLine(String line) {
        log.info("Parsing line: [{}].", line);
        String[] parts = line.split("\",\"");
        if (parts.length < 6) {
            warn("Invalid line: Less than 6 columns.");
            return Optional.empty();
        }
        parts[0] = parts[0].replaceFirst("^\"", "");
        parts[parts.length-1] = parts[parts.length-1].replaceFirst("\"$", "");

        log.debug("Parts({}): {}", parts.length, parts);
<commit_msg>Make WorldIndexValueRetriever less verbose<commit_after>package ee.tuleva.onboarding.comparisons.fundvalue.retrieval;

import com.rollbar.Rollbar;
import ee.tuleva.onboarding.comparisons.fundvalue.ComparisonFund;
import ee.tuleva.onboarding.comparisons.fundvalue.FundValue;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Slf4j
@Service
@RequiredArgsConstructor
public class WorldIndexValueRetriever implements FundValueRetriever {

    private final Environment environment;
    private final RestTemplate restTemplate;

    @Value("${logging.rollbar.accessToken:#{null}}")
    private String accessToken;

    private static final String SOURCE_URL = "https://docs.google.com/spreadsheets/d/125aXusxnf-Mij-4D4W5Qnneb8H4chjebnIMpyrAfRkI/gviz/tq?tqx=out:csv&gid=619370394";

    @Override
    public ComparisonFund getRetrievalFund() {
        return ComparisonFund.MARKET;
    }

    @Override
    public List<FundValue> retrieveValuesForRange(Instant startDate, Instant endDate) {
        return restTemplate.execute(SOURCE_URL, HttpMethod.GET, getRequestCallback(), getResponseExtractor(startDate, endDate));
    }

    private ResponseExtractor<List<FundValue>> getResponseExtractor(Instant startDate, Instant endDate) {
        return response -> {
            if (response.getStatusCode() == HttpStatus.OK) {
                InputStream body = response.getBody();
                BufferedReader reader = new BufferedReader(new InputStreamReader(body, StandardCharsets.UTF_8));
                Stream<String> lines = reader.lines().skip(3);
                return parseLines(lines, startDate, endDate);
            } else {
                warn("Failed to fetch World Index values");
                return Collections.emptyList();
            }
        };
    }

    private List<FundValue> parseLines(Stream<String> lines, Instant startDate, Instant endDate) {
        return lines
            .map(this::parseLine)
            .filter(Optional::isPresent)
            .map(Optional::get)
            .filter((FundValue value) -> {
                Instant time = value.getTime();
                return (startDate.isBefore(time) || startDate.equals(time)) && (endDate.isAfter(time) || endDate.equals(time));
            })
            .collect(Collectors.toList());
    }

    private Optional<FundValue> parseLine(String line) {
        log.debug("Parsing line: [{}].", line);
        String[] parts = line.split("\",\"");
        if (parts.length < 6) {
            warn("Invalid line: Less than 6 columns.");
            return Optional.empty();
        }
        parts[0] = parts[0].replaceFirst("^\"", "");
        parts[parts.length-1] = parts[parts.length-1].replaceFirst("\"$", "");

        log.debug("Parts({}): {}", parts.length, parts);

{'fraction_non_alphanumeric': 0.11502284332168772, 'fraction_numerical': 0.007524858908895459, 'mean_word_length': 3.5887792848335387, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6546288', 'n_tokens_mistral': 2188, 'n_tokens_neox': 2098, 'n_words': 393}