<commit_before>package com.northpine.scrape;

import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import com.northpine.scrape.ogr.GeoCollector;
import com.northpine.scrape.ogr.OgrCollector;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import static com.northpine.scrape.JobManager.MAN;
import static com.northpine.scrape.request.HttpRequester.Q;
import static java.lang.String.format;
import static java.util.concurrent.CompletableFuture.runAsync;

/**
 * Scrapes ArcGIS REST Servers
 */
public class ScrapeJob {

  private static final int CHUNK_SIZE = 200;



  private static final String OUTPUT_FOLDER = "output";



  private static final Logger log = LoggerFactory.getLogger( ScrapeJob.class );

  private final ExecutorService executor;

  private String layerName;

  private AtomicInteger current;

  private AtomicInteger done;

  private AtomicInteger total;

  private AtomicBoolean failed;

  private boolean isDone;

  private String outputFileBase;

  private String outputZip;

  private String layerUrl;

  private String queryUrlStr;

  private String failMessage;

  private File zipFile;


  /**
   * @param layerUrl Does not include "/query" appended to end of url to layer.
   */
  public ScrapeJob(String layerUrl) {
    executor = Executors.newWorkStealingPool();
    current = new AtomicInteger();
    total = new AtomicInteger();
    failed = new AtomicBoolean( false);
    this.layerUrl = layerUrl ;
    this.queryUrlStr = layerUrl + "/query";
    this.layerName = getLayerName();
    this.outputFileBase =  OUTPUT_FOLDER + "/" + layerName;
    this.outputZip =  OUTPUT_FOLDER + "/" + layerName + ".zip";
  }

  public void startScraping() {
    current = new AtomicInteger();
    done = new AtomicInteger();
    isDone = false;
    URL queryUrl = getURL( queryUrlStr + "?where=1=1&returnIdsOnly=true&f=json&outSR=3857" );
    JSONObject idsJson = Q.submitSyncRequest(queryUrl.toString())
        .orElseThrow(RuntimeException::new);
    JSONArray arr = idsJson.getJSONArray( "objectIds" );
    OgrCollector collector = new GeoCollector(outputFileBase);


    buildIdStrs( arr ).stream()
        .map( idListStr -> "OBJECTID in (" + idListStr + ")" )
        .forEach(whereClause -> {
          try {
            String file = outputFileBase + current.incrementAndGet() + ".json";
            var request = Unirest.get(queryUrlStr)
                .queryString("where", whereClause)
                .queryString("outFields", "*")
                .queryString("f", "json")
                .asBinary();
            if(request.getStatus() == 200) {
              writeToFile(request.getRawBody(), file);
              var wasSuccessful = collector.addJsonToPool(file);
              if(wasSuccessful) {
                var numDone = done.incrementAndGet();
                var numTotal = total.get();
                if(numDone % (numTotal / 10) == 0) {
                  log.info(format("%d/%d requests done for %s", numDone, numTotal, layerUrl));
                }
              } else {
                failJob("Couldn't run ogr2ogr");
              }
            } else {
              throw new RuntimeException(request.getStatusText());
            }
          } catch (UnirestException httpException) {
            log.error("Couldn't connect to server", httpException);
            failJob("Connecting to server for query failed");
          }
        });

    zipFile = collector.zipUpPool();
    isDone = true;
    log.info("Zipped '" + outputZip + "'");
    log.info("Done with job.");
  }




  public int getNumDone() {
    if(done != null) {
      return done.get();
    } else {
      return -1;
    }
  }

  public void stopJob() {
    executor.shutdownNow();
  }

  public String getName() {
    return layerName;<commit_msg>add 100ms throttle<commit_after>package com.northpine.scrape;

import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import com.northpine.scrape.ogr.GeoCollector;
import com.northpine.scrape.ogr.OgrCollector;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import static com.northpine.scrape.JobManager.MAN;
import static com.northpine.scrape.request.HttpRequester.Q;
import static java.lang.String.format;
import static java.util.concurrent.CompletableFuture.runAsync;

/**
 * Scrapes ArcGIS REST Servers
 */
public class ScrapeJob {

  private static final int CHUNK_SIZE = 200;



  private static final String OUTPUT_FOLDER = "output";



  private static final Logger log = LoggerFactory.getLogger( ScrapeJob.class );

  private final ExecutorService executor;

  private String layerName;

  private AtomicInteger current;

  private AtomicInteger done;

  private AtomicInteger total;

  private AtomicBoolean failed;

  private boolean isDone;

  private String outputFileBase;

  private String outputZip;

  private String layerUrl;

  private String queryUrlStr;

  private String failMessage;

  private File zipFile;


  /**
   * @param layerUrl Does not include "/query" appended to end of url to layer.
   */
  public ScrapeJob(String layerUrl) {
    executor = Executors.newWorkStealingPool();
    current = new AtomicInteger();
    total = new AtomicInteger();
    failed = new AtomicBoolean( false);
    this.layerUrl = layerUrl ;
    this.queryUrlStr = layerUrl + "/query";
    this.layerName = getLayerName();
    this.outputFileBase =  OUTPUT_FOLDER + "/" + layerName;
    this.outputZip =  OUTPUT_FOLDER + "/" + layerName + ".zip";
  }

  public void startScraping() {
    current = new AtomicInteger();
    done = new AtomicInteger();
    isDone = false;
    URL queryUrl = getURL( queryUrlStr + "?where=1=1&returnIdsOnly=true&f=json&outSR=3857" );
    JSONObject idsJson = Q.submitSyncRequest(queryUrl.toString())
        .orElseThrow(RuntimeException::new);
    JSONArray arr = idsJson.getJSONArray( "objectIds" );
    OgrCollector collector = new GeoCollector(outputFileBase);


    buildIdStrs( arr ).stream()
        .map( idListStr -> "OBJECTID in (" + idListStr + ")" )
        .forEach(whereClause -> {
          try {
            String file = outputFileBase + current.incrementAndGet() + ".json";
            var request = Unirest.get(queryUrlStr)
                .queryString("where", whereClause)
                .queryString("outFields", "*")
                .queryString("f", "json")
                .asBinary();
            if(request.getStatus() == 200) {
              writeToFile(request.getRawBody(), file);
              var wasSuccessful = collector.addJsonToPool(file);
              if(wasSuccessful) {
                var numDone = done.incrementAndGet();
                var numTotal = total.get();
                if(numDone % (numTotal / 10) == 0) {
                  log.info(format("%d/%d requests done for %s", numDone, numTotal, layerUrl));
                }
              } else {
                failJob("Couldn't run ogr2ogr");
              }
              Thread.sleep(100);
            } else {
              throw new RuntimeException(request.getStatusText());
            }
          } catch (UnirestException | InterruptedException httpException) {
            log.error("Couldn't connect to server", httpException);
            failJob("Connecting to server for query failed");
          }
        });

    zipFile = collector.zipUpPool();
    isDone = true;
    log.info("Zipped '" + outputZip + "'");
    log.info("Done with job.");
  }




  public int getNumDone() {
    if(done != null) {
      return done.get();
    } else {
      return -1;
    }
  }

  public void stopJob() {
    executor.shutdownNow();
  }

  public String getName() {
    return layerName;
{'fraction_non_alphanumeric': 0.11014271707157643, 'fraction_numerical': 0.004793550495696699, 'mean_word_length': 2.955191727703576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1881990', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2664, 'n_words': 611}