<commit_before>package au.gov.ga.geodesy.domain.service;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.Scanner;
import java.util.StringTokenizer;

import javax.annotation.PostConstruct;

import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;

import au.gov.ga.geodesy.domain.model.Event;
import au.gov.ga.geodesy.domain.model.EventPublisher;
import au.gov.ga.geodesy.domain.model.EventSubscriber;
import au.gov.ga.geodesy.domain.model.Position;
import au.gov.ga.geodesy.domain.model.PositionRepository;
import au.gov.ga.geodesy.domain.model.WeeklySolution;
import au.gov.ga.geodesy.domain.model.WeeklySolutionAvailable;
import au.gov.ga.geodesy.domain.model.WeeklySolutionRepository;

@Component
@Transactional("geodesyTransactionManager")
public class PositionService implements EventSubscriber<WeeklySolutionAvailable> {

    @Autowired
    private WeeklySolutionRepository solutions;

    @Autowired
    private EventPublisher eventPublisher;

    @Autowired
    private PositionRepository positions;

    @PostConstruct
    public void subscribe() {
        eventPublisher.subscribe(this);
    }

    public boolean canHandle(Event event) {
        return event instanceof WeeklySolutionAvailable;
    }

    public void handle(WeeklySolutionAvailable event) {
        WeeklySolution solution = solutions.findOne(event.getWeeklySolutionId());

        File sinex = new File(solution.getSinexFile());
        try (
            Scanner scanner = new Scanner(sinex);
        ) {
            while (!scanner.nextLine().startsWith("+SOLUTION/ESTIMATE"));
            scanner.nextLine();

            String line = null;
            while (!(line = scanner.nextLine()).startsWith("-SOLUTION/ESTIMATE")) {

                Pair<String, BigDecimal> xLine = parseEstimatedValue(line);
                Pair<String, BigDecimal> yLine = parseEstimatedValue(scanner.nextLine());

                @SuppressWarnings("unused")
                Pair<String, BigDecimal> zLine = parseEstimatedValue(scanner.nextLine());

                double x = xLine.getRight().doubleValue();
                double y = yLine.getRight().doubleValue();

                Position p = new Position(xLine.getLeft(), null, 5332, x, y, solution.getEpoch(), solution.getAsAt(), solution.getId());
                positions.save(p);
            }
            System.out.println("last line read is: " + line);


        } catch (IOException e) {
            e.printStackTrace();
        }
        eventPublisher.handled(event);
    }

    private Pair<String, BigDecimal> parseEstimatedValue(String line) {
        StringTokenizer tokenizer = new StringTokenizer(line);

        drop(tokenizer, 2);
        String fourCharacterId = tokenizer.nextToken();

        drop(tokenizer, 5);
        BigDecimal value = new BigDecimal(tokenizer.nextToken());

        return Pair.of(fourCharacterId, value);
    }

    private void drop(StringTokenizer tokenizer, int n) {
        for (int i = 0; i < n; i++) {
            tokenizer.nextToken();
        }
    }
}
<commit_msg>Decode SINEX files using iso-8859-1.<commit_after>package au.gov.ga.geodesy.domain.service;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.Scanner;
import java.util.StringTokenizer;

import javax.annotation.PostConstruct;

import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;

import au.gov.ga.geodesy.domain.model.Event;
import au.gov.ga.geodesy.domain.model.EventPublisher;
import au.gov.ga.geodesy.domain.model.EventSubscriber;
import au.gov.ga.geodesy.domain.model.Position;
import au.gov.ga.geodesy.domain.model.PositionRepository;
import au.gov.ga.geodesy.domain.model.WeeklySolution;
import au.gov.ga.geodesy.domain.model.WeeklySolutionAvailable;
import au.gov.ga.geodesy.domain.model.WeeklySolutionRepository;

@Component
@Transactional("geodesyTransactionManager")
public class PositionService implements EventSubscriber<WeeklySolutionAvailable> {

    @Autowired
    private WeeklySolutionRepository solutions;

    @Autowired
    private EventPublisher eventPublisher;

    @Autowired
    private PositionRepository positions;

    @PostConstruct
    public void subscribe() {
        eventPublisher.subscribe(this);
    }

    public boolean canHandle(Event event) {
        return event instanceof WeeklySolutionAvailable;
    }

    public void handle(WeeklySolutionAvailable event) {
        WeeklySolution solution = solutions.findOne(event.getWeeklySolutionId());

        File sinex = new File(solution.getSinexFile());
        try (
            Scanner scanner = new Scanner(sinex, "iso-8859-1");
        ) {
            while (!scanner.nextLine().startsWith("+SOLUTION/ESTIMATE"));
            scanner.nextLine();

            String line = null;
            while (!(line = scanner.nextLine()).startsWith("-SOLUTION/ESTIMATE")) {

                Pair<String, BigDecimal> xLine = parseEstimatedValue(line);
                Pair<String, BigDecimal> yLine = parseEstimatedValue(scanner.nextLine());

                @SuppressWarnings("unused")
                Pair<String, BigDecimal> zLine = parseEstimatedValue(scanner.nextLine());

                double x = xLine.getRight().doubleValue();
                double y = yLine.getRight().doubleValue();

                Position p = new Position(xLine.getLeft(), null, 5332, x, y, solution.getEpoch(), solution.getAsAt(), solution.getId());
                positions.save(p);
            }
            System.out.println("last line read is: " + line);


        } catch (IOException e) {
            e.printStackTrace();
        }
        eventPublisher.handled(event);
    }

    private Pair<String, BigDecimal> parseEstimatedValue(String line) {
        StringTokenizer tokenizer = new StringTokenizer(line);

        drop(tokenizer, 2);
        String fourCharacterId = tokenizer.nextToken();

        drop(tokenizer, 5);
        BigDecimal value = new BigDecimal(tokenizer.nextToken());

        return Pair.of(fourCharacterId, value);
    }

    private void drop(StringTokenizer tokenizer, int n) {
        for (int i = 0; i < n; i++) {
            tokenizer.nextToken();
        }
    }
}

{'fraction_non_alphanumeric': 0.10843927094501456, 'fraction_numerical': 0.003982233113799969, 'mean_word_length': 3.1043368950345696, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '982179', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1803, 'n_words': 351}