<commit_before>package au.gov.amsa.geo.adhoc;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.util.zip.GZIPOutputStream;

import org.apache.log4j.Logger;

import au.gov.amsa.risky.format.BinaryFixes;
import au.gov.amsa.risky.format.BinaryFixesFormat;
import rx.Observable;

public class BinaryFixesWithMmsiGzSorterMain {

    private static final Logger log = Logger.getLogger(BinaryFixesWithMmsiGzSorterMain.class);

    public static void main(String[] args) {

        File tracksDir = new File("/home/dxm/AIS/tracks");
        File sortedTracks = new File("/home/dxm/AIS/tracks-sorted");

        BinaryFixes.from(new File(tracksDir, "2019-01-01.track.gz"), true, BinaryFixesFormat.WITH_MMSI) //
                .doOnNext(System.out::println) //
                .subscribe();

        sortedTracks.mkdir();
        Observable<File> files = Observable //
                .from(tracksDir.listFiles()) //
                .filter(x -> x.getName().endsWith(".track.gz"));
        files.flatMap(x -> Observable //
                .just(x) //
                .doOnNext(f -> log.info("sorting " + f)) //
                .doOnNext(f -> sort(f, sortedTracks))) //
                .doOnNext(f -> log.info("sorted " + f)) //
                .toBlocking() //
                .subscribe();
    }

    private static void sort(File file, File outputDir) {
        File sorted = new File(outputDir, file.getName());
        try (OutputStream out = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(sorted)))) {
            BinaryFixes.from(file, true, BinaryFixesFormat.WITH_MMSI) //
                    .doOnNext(System.out::println) //
                    .toSortedList((x, y) -> Long.compare(x.time(), y.time())) //
                    .flatMap(x -> Observable.from(x)) //<commit_msg>add logging<commit_after>package au.gov.amsa.geo.adhoc;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPOutputStream;

import org.apache.log4j.Logger;

import au.gov.amsa.risky.format.BinaryFixes;
import au.gov.amsa.risky.format.BinaryFixesFormat;
import rx.Observable;
import rx.schedulers.Schedulers;

public class BinaryFixesWithMmsiGzSorterMain {

    private static final Logger log = Logger.getLogger(BinaryFixesWithMmsiGzSorterMain.class);

    public static void main(String[] args) {

        File tracksDir = new File("/home/dxm/AIS/tracks");
        File sortedTracks = new File("/home/dxm/AIS/tracks-sorted");

        sortedTracks.mkdir();
        Observable<File> files = Observable //
                .from(tracksDir.listFiles()) //
                .filter(x -> x.getName().endsWith(".track.gz"));
        AtomicInteger n = new AtomicInteger();
        files.flatMap(x -> Observable //
                .just(x) //
                .observeOn(Schedulers.computation()) //
                .doOnNext(f -> log.info(n.incrementAndGet() + ": sorting " + f.getName())) //
                .doOnNext(f -> sort(f, sortedTracks))) //
                .doOnNext(f -> log.info(n.get() + ": sorted " + f.getName())) //
                .toBlocking() //
                .subscribe();
    }

    private static void sort(File file, File outputDir) {
        File sorted = new File(outputDir, file.getName());
        try (OutputStream out = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(sorted)))) {
            BinaryFixes.from(file, true, BinaryFixesFormat.WITH_MMSI) //
                    .toSortedList((x, y) -> Long.compare(x.time(), y.time())) //
                    .flatMap(x -> Observable.from(x)) //
{'fraction_non_alphanumeric': 0.13833075734157652, 'fraction_numerical': 0.002575991756826378, 'mean_word_length': 2.4669642857142855, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2374790', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1160, 'n_words': 210}