<commit_before>package org.allenai.datastore

import java.io.InputStream
import java.net.{ URI, URL }
import java.nio.ByteBuffer
import java.nio.channels.{ Channels, ReadableByteChannel, WritableByteChannel }
import java.nio.file._
import java.nio.file.attribute.BasicFileAttributes
import java.util.zip.{ ZipEntry, ZipFile, ZipOutputStream }

import com.amazonaws.AmazonServiceException
import com.amazonaws.auth.BasicAWSCredentials
import com.amazonaws.event.{ ProgressEvent, ProgressListener }
import com.amazonaws.services.s3.AmazonS3Client
import com.amazonaws.services.s3.model._
import com.amazonaws.services.s3.transfer.TransferManager
import org.allenai.common.{ Logging, Resource }
import org.apache.commons.io.FileUtils
import org.slf4j.{ Logger, LoggerFactory }
import ch.qos.logback.classic.Level

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.util.control.NonFatal
import scala.util.{ Success, Failure, Try, Random }

/** Represents a datastore
  *
  * This is a thin layer over an S3 bucket that stores the data. Data is identified by group
  * ("org.allenai.something"), name ("WordNet"), and version (an integer). It supports files as well
  * as directories.
  *
  * Items are published to the datastore, and then referred to with the *path() methods. All data is
  * cached, so access to all items should be very fast, except for the first time.
  *
  * It might make more sense to get Datastore objects from the companion object, rather than
  * creating them here.
  *
  * @param name name of the datastore. Corresponds to the name of the bucket in S3. Currently we
  *      have "public" and "private".
  * @param s3   properly authenticated S3 client.
  */
class Datastore(val name: String, val s3: AmazonS3Client) extends Logging {

  private val random = new Random
  @tailrec
  private def withRetries[T](activity: String, retries: Int = 10)(f: => T): T =
    if (retries <= 0) {
      f
    } else {
      val sleepTime = random.nextInt(10000) + 10000 // sleep between 10 and 20 seconds
      // If something goes wrong, we sleep a random amount of time, to make sure that we don't slam<commit_msg>Removed @tailrec because of assumed scale compiler bug<commit_after>package org.allenai.datastore

import java.io.InputStream
import java.net.{ URI, URL }
import java.nio.ByteBuffer
import java.nio.channels.{ Channels, ReadableByteChannel, WritableByteChannel }
import java.nio.file._
import java.nio.file.attribute.BasicFileAttributes
import java.util.zip.{ ZipEntry, ZipFile, ZipOutputStream }

import com.amazonaws.AmazonServiceException
import com.amazonaws.auth.BasicAWSCredentials
import com.amazonaws.event.{ ProgressEvent, ProgressListener }
import com.amazonaws.services.s3.AmazonS3Client
import com.amazonaws.services.s3.model._
import com.amazonaws.services.s3.transfer.TransferManager
import org.allenai.common.{ Logging, Resource }
import org.apache.commons.io.FileUtils
import org.slf4j.{ Logger, LoggerFactory }
import ch.qos.logback.classic.Level

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.util.control.NonFatal
import scala.util.{ Success, Failure, Try, Random }

/** Represents a datastore
  *
  * This is a thin layer over an S3 bucket that stores the data. Data is identified by group
  * ("org.allenai.something"), name ("WordNet"), and version (an integer). It supports files as well
  * as directories.
  *
  * Items are published to the datastore, and then referred to with the *path() methods. All data is
  * cached, so access to all items should be very fast, except for the first time.
  *
  * It might make more sense to get Datastore objects from the companion object, rather than
  * creating them here.
  *
  * @param name name of the datastore. Corresponds to the name of the bucket in S3. Currently we
  *      have "public" and "private".
  * @param s3   properly authenticated S3 client.
  */
class Datastore(val name: String, val s3: AmazonS3Client) extends Logging {

  private val random = new Random
  private def withRetries[T](activity: String, retries: Int = 10)(f: => T): T =
    if (retries <= 0) {
      f
    } else {
      val sleepTime = random.nextInt(10000) + 10000 // sleep between 10 and 20 seconds
      // If something goes wrong, we sleep a random amount of time, to make sure that we don't slam
{'fraction_non_alphanumeric': 0.08807339449541285, 'fraction_numerical': 0.012844036697247707, 'mean_word_length': 4.100584795321637, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1563375', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1283, 'n_words': 467}