<commit_before>package cosbench_ng



import org.slf4j.LoggerFactory
import com.amazonaws.ClientConfiguration
import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder }
import com.amazonaws.services.s3.model.{ GetObjectRequest, ObjectMetadata }
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials}

import java.util.concurrent.Executors
import java.io.BufferedReader;
import java.io.InputStreamReader;

import scala.concurrent.ExecutionContext
import scala.util.{ Try, Failure, Success }
import scala.concurrent.{Future, blocking}
import java.io.InputStream
import org.apache.http.conn.ssl.{SSLConnectionSocketFactory }
import org.apache.http.ssl.{SSLContextBuilder, TrustStrategy}
import javax.net.ssl.{HostnameVerifier, SSLSession}
import java.security.cert.X509Certificate


object GetS3Client {
  val log = LoggerFactory.getLogger(this.getClass)

  def init(c:Config) = get(c)
  
  def get(c: Config) = {
    if (s3Client.isEmpty)
      s3Client = Some(createS3Client(c))

    s3Client.get
  }

  def test(bkt: String) : Boolean =
    Try {
      s3Client.get.listObjects(bkt)
    } match {
      case Success(e) => true
      case Failure(e) =>
        log.error(e.toString)
        false
    }

  private var s3Client: Option[AmazonS3] = None

  private def sslNoVerifyConnFactory(): SSLConnectionSocketFactory = {
    // No ssl verification

    // from: http://literatejava.com/networks/ignore-ssl-certificate-errors-apache-httpclient-4-4/
    // setup a trust strategy that allows all certificates
    val sslContext = SSLContextBuilder.create()
      .useProtocol("SSL")
      .loadTrustMaterial(null, new TrustStrategy() {
        def isTrusted(arg0: Array[X509Certificate], arg1: String) = true
      })
      .build()

    val sslConFactory = new SSLConnectionSocketFactory(
      sslContext,
      new HostnameVerifier { def verify(hostname: String, session: SSLSession) = true })

    return sslConFactory
  }

  private def createS3Client(c: Config): AmazonS3 = {
    
    require (c.aidSkey._1 != "aid")
    val awsCredentials = new BasicAWSCredentials(c.aidSkey._1, c.aidSkey._2)

    val clientConfig = new ClientConfiguration().withMaxErrorRetry(0)
    
    // No SSL-Verify
    clientConfig.getApacheHttpClientConfig()
      .setSslSocketFactory(sslNoVerifyConnFactory())

    // S3 client with retries disabled
    val s3Client = AmazonS3ClientBuilder
      .standard()
      .withEndpointConfiguration(new EndpointConfiguration(c.endpoint, c.region))
      .withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
      .withClientConfiguration(clientConfig)
      .withPathStyleAccessEnabled(true)
      .build()

    Try {
      s3Client.listBuckets()
    } match {
      case Success(e) => true
      case Failure(e) =>
        log.error("Problem with S3 configuration, unable to do a test list-bucket. ")
        log.error("Using AID        = " + awsCredentials.getAWSAccessKeyId())
        log.error("Using secret key = " + awsCredentials.getAWSSecretKey())
        log.error("Using endpoint   = " + c.endpoint)
        log.error(e.toString)<commit_msg>fixed #1<commit_after>package cosbench_ng



import org.slf4j.LoggerFactory
import com.amazonaws.ClientConfiguration
import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder }
import com.amazonaws.services.s3.model.{ GetObjectRequest, ObjectMetadata }
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials}

import java.util.concurrent.Executors
import java.io.BufferedReader;
import java.io.InputStreamReader;

import scala.concurrent.ExecutionContext
import scala.util.{ Try, Failure, Success }
import scala.concurrent.{Future, blocking}
import java.io.InputStream
import org.apache.http.conn.ssl.{SSLConnectionSocketFactory }
import org.apache.http.ssl.{SSLContextBuilder, TrustStrategy}
import javax.net.ssl.{HostnameVerifier, SSLSession}
import java.security.cert.X509Certificate


object GetS3Client {
  val log = LoggerFactory.getLogger(this.getClass)

  def init(c:Config) = get(c)
  
  def get(c: Config) = {
    if (s3Client.isEmpty)
      s3Client = Some(createS3Client(c))

    s3Client.get
  }

  def test(bkt: String) : Boolean =
    Try {
      s3Client.get.listObjects(bkt)
    } match {
      case Success(e) => true
      case Failure(e) =>
        log.error(e.toString)
        false
    }

  private var s3Client: Option[AmazonS3] = None

  private def sslNoVerifyConnFactory(): SSLConnectionSocketFactory = {
    // No ssl verification

    // from: http://literatejava.com/networks/ignore-ssl-certificate-errors-apache-httpclient-4-4/
    // setup a trust strategy that allows all certificates
    val sslContext = SSLContextBuilder.create()
      .useProtocol("SSL")
      .loadTrustMaterial(null, new TrustStrategy() {
        def isTrusted(arg0: Array[X509Certificate], arg1: String) = true
      })
      .build()

    val sslConFactory = new SSLConnectionSocketFactory(
      sslContext,
      new HostnameVerifier { def verify(hostname: String, session: SSLSession) = true })

    return sslConFactory
  }

  private def createS3Client(c: Config): AmazonS3 = {
    
    require (c.aidSkey._1 != "aid")
    val awsCredentials = new BasicAWSCredentials(c.aidSkey._1, c.aidSkey._2)

    val clientConfig = new ClientConfiguration().withMaxErrorRetry(0)
    
    // No SSL-Verify
    clientConfig.getApacheHttpClientConfig()
      .setSslSocketFactory(sslNoVerifyConnFactory())

    // S3 client with retries disabled
    val s3Client = AmazonS3ClientBuilder
      .standard()
      .withEndpointConfiguration(new EndpointConfiguration(c.endpoint, c.region))
      .withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
      .withClientConfiguration(clientConfig)
      .withPathStyleAccessEnabled(true)
      .build()

    Try {
      s3Client.listObjects(c.bucketName)
    } match {
      case Success(e) => true
      case Failure(e) =>
        log.error("Problem with S3 configuration, unable to do a test list objects on bucket: " + c.bucketName)
        log.error("Using AID        = " + awsCredentials.getAWSAccessKeyId())
        log.error("Using secret key = " + awsCredentials.getAWSSecretKey())
        log.error("Using endpoint   = " + c.endpoint)
        log.error(e.toString)
{'fraction_non_alphanumeric': 0.10113406866552742, 'fraction_numerical': 0.010719279167313966, 'mean_word_length': 3.634989200863931, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5431569', 'n_tokens_mistral': 1949, 'n_tokens_neox': 1934, 'n_words': 396}