<commit_before>package outbackcdx;


import java.io.*;
import java.util.Base64;

import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.rocksdb.RocksDBException;
import org.rocksdb.WriteBatch;

import static outbackcdx.Json.GSON;


public class ChangePollingThread extends Thread {
    final byte[] SEQ_NUM_KEY = "#ReplicationSequence".getBytes();

    String primaryReplicationUrl = null;
    int pollingInterval = 10;
    DataStore dataStore = null;
    Index index = null;
    Long sequenceNumber = Long.valueOf(0);
    String finalUrl = null;
    String collection;

    ChangePollingThread(String primaryReplicationUrl, int pollingInterval, DataStore dataStore) throws IOException {
        this.primaryReplicationUrl = primaryReplicationUrl;
        this.pollingInterval = pollingInterval;
        this.dataStore = dataStore;
        this.primaryReplicationUrl = this.primaryReplicationUrl.replaceFirst("/$", "");
        String[] splitCollectionUrl = this.primaryReplicationUrl.split("/");
        collection = splitCollectionUrl[splitCollectionUrl.length - 1];
        this.index = dataStore.getIndex(collection, true);
        this.primaryReplicationUrl = primaryReplicationUrl + "/changes?since=";
    }

    public void run() {
        while (true) {
            long startTime = System.currentTimeMillis();
            try {
                byte[] output = this.index.db.get(SEQ_NUM_KEY);
                String sequence = output.toString();
                sequenceNumber = Long.valueOf(sequence);
            } catch (RocksDBException e) {
                System.out.println("Received rocks db exception while looking up the value of the key " + SEQ_NUM_KEY.toString() + " locally");
                e.printStackTrace();
            } catch (Exception e) {
                System.out.println("Received an exception while looking up the value of the key " + SEQ_NUM_KEY.toString() + " locally");
                e.printStackTrace();
            }
            finalUrl = primaryReplicationUrl + sequenceNumber;
            try {
                System.out.println("Polling " + finalUrl + " for changes since sequence number " + sequenceNumber);
                System.out.println("Beginning replication of changes into collection '" + collection + "'...");
                replicate();
                System.out.println("Replication complete.");
            } catch (IOException e) {
                System.out.println("Received I/O exception while processing " + finalUrl);
                e.printStackTrace();
            } catch (RocksDBException e){
                System.out.println("The plane has crashed into the mountain. RocksDB threw an exception during replication from "+ finalUrl);<commit_msg>new String(byte[]) instead of byte[].toString per @ato<commit_after>package outbackcdx;


import java.io.*;
import java.util.Base64;

import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.rocksdb.RocksDBException;
import org.rocksdb.WriteBatch;

import static outbackcdx.Json.GSON;


public class ChangePollingThread extends Thread {
    final byte[] SEQ_NUM_KEY = "#ReplicationSequence".getBytes();

    String primaryReplicationUrl = null;
    int pollingInterval = 10;
    DataStore dataStore = null;
    Index index = null;
    Long sequenceNumber = Long.valueOf(0);
    String finalUrl = null;
    String collection;

    ChangePollingThread(String primaryReplicationUrl, int pollingInterval, DataStore dataStore) throws IOException {
        this.primaryReplicationUrl = primaryReplicationUrl;
        this.pollingInterval = pollingInterval;
        this.dataStore = dataStore;
        this.primaryReplicationUrl = this.primaryReplicationUrl.replaceFirst("/$", "");
        String[] splitCollectionUrl = this.primaryReplicationUrl.split("/");
        collection = splitCollectionUrl[splitCollectionUrl.length - 1];
        this.index = dataStore.getIndex(collection, true);
        this.primaryReplicationUrl = primaryReplicationUrl + "/changes?since=";
    }

    public void run() {
        while (true) {
            long startTime = System.currentTimeMillis();
            try {
                byte[] output = this.index.db.get(SEQ_NUM_KEY);
                String sequence = output.toString();
                sequenceNumber = Long.valueOf(sequence);
            } catch (RocksDBException e) {
                System.out.println("Received rocks db exception while looking up the value of the key " + new String(SEQ_NUM_KEY) + " locally");
                e.printStackTrace();
            } catch (Exception e) {
                System.out.println("Received an exception while looking up the value of the key " + new String(SEQ_NUM_KEY) + " locally");
                e.printStackTrace();
            }
            finalUrl = primaryReplicationUrl + sequenceNumber;
            try {
                System.out.println("Polling " + finalUrl + " for changes since sequence number " + sequenceNumber);
                System.out.println("Beginning replication of changes into collection '" + collection + "'...");
                replicate();
                System.out.println("Replication complete.");
            } catch (IOException e) {
                System.out.println("Received I/O exception while processing " + finalUrl);
                e.printStackTrace();
            } catch (RocksDBException e){
                System.out.println("The plane has crashed into the mountain. RocksDB threw an exception during replication from "+ finalUrl);
{'fraction_non_alphanumeric': 0.09870304867778339, 'fraction_numerical': 0.00202122283981809, 'mean_word_length': 2.7629911280101394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964735', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1483, 'n_words': 375}