<commit_before>from . import base
from . import messages as deployment_messages
from boto.gs import key
from boto.s3 import connection
from gcloud import storage
from protorpc import messages
import boto
import cStringIO
import dns.resolver
import logging
import mimetypes
import os
import webapp2


class TestCase(base.DestinationTestCase):

  def test_domain_cname_is_gcs(self):
    bucket_name = self.deployment.config.bucket
    CNAME = 'c.storage.googleapis.com'

    message = deployment_messages.TestResultMessage()
    message.title = 'CNAME for {} is {}'.format(bucket_name, CNAME)

    dns_resolver = dns.resolver.Resolver()
    dns_resolver.nameservers = ['8.8.8.8']  # Use Google's DNS.

    try:
      content = str(dns_resolver.query(bucket_name, 'CNAME')[0])
    except:
      text = "Can't verify CNAME for {} is mapped to {}"
      message.result = deployment_messages.Result.WARNING
      message.text = text.format(bucket_name, CNAME)
    if not content.startswith(CNAME):
      text = 'CNAME mapping for {} is not GCS! Found {}, expected {}'
      message.result = deployment_messages.Result.WARNING
      message.text = text.format(bucket_name, content, CNAME)
    else:
      text = 'CNAME for {} -> {}'.format(bucket_name, content, CNAME)
      message.text = text.format(text, content, CNAME)
    return message


class Config(messages.Message):
  bucket = messages.StringField(1)
  access_key = messages.StringField(2)
  access_secret = messages.StringField(3)
  project = messages.StringField(4)
  email = messages.StringField(5)
  key_path = messages.StringField(6)


class GoogleCloudStorageDestination(base.BaseDestination):
  NAME = 'gcs'
  TestCase = TestCase
  Config = Config

  def __str__(self):
    return 'gs://{}'.format(self.config.bucket)

  @property
  def use_interoperable_auth(self):
    return self.config.access_key is not None

  @webapp2.cached_property
  def bucket(self):
    if self.use_interoperable_auth:
      gs_connection = boto.connect_gs(
          self.config.access_key, self.config.access_secret,
          calling_format=connection.OrdinaryCallingFormat())
    else:
      gs_connection = storage.get_connection(
          self.config.project, self.config.email, self.config.key_path)
    return gs_connection.get_bucket(self.config.bucket)

  def prelaunch(self, dry_run=False):
    if dry_run:
      return
    logging.info('Configuring GS bucket: {}'.format(self.config.bucket))
    if self.use_interoperable_auth:
      self.bucket.set_acl('public-read')
      self.bucket.configure_versioning(False)
      self.bucket.configure_website(main_page_suffix='index.html', error_key='<KEY>')
    else:
      acl = self.bucket.get_default_object_acl()
      acl.all().grant_read().revoke_write()
      acl.save()
      self.bucket.configure_website(main_page_suffix='index.html', not_found_page='404.html')

  def write_control_file(self, path, content):
    path = os.path.join(self.control_dir, path.lstrip('/'))
    return self.write_file(path, content, policy='private')

  def read_file(self, path):<commit_msg>Update use_interoperable_auth detection.<commit_after>from . import base
from . import messages as deployment_messages
from boto.gs import key
from boto.s3 import connection
from gcloud import storage
from protorpc import messages
import boto
import cStringIO
import dns.resolver
import logging
import mimetypes
import os
import webapp2


class TestCase(base.DestinationTestCase):

  def test_domain_cname_is_gcs(self):
    bucket_name = self.deployment.config.bucket
    CNAME = 'c.storage.googleapis.com'

    message = deployment_messages.TestResultMessage()
    message.title = 'CNAME for {} is {}'.format(bucket_name, CNAME)

    dns_resolver = dns.resolver.Resolver()
    dns_resolver.nameservers = ['8.8.8.8']  # Use Google's DNS.

    try:
      content = str(dns_resolver.query(bucket_name, 'CNAME')[0])
    except:
      text = "Can't verify CNAME for {} is mapped to {}"
      message.result = deployment_messages.Result.WARNING
      message.text = text.format(bucket_name, CNAME)
    if not content.startswith(CNAME):
      text = 'CNAME mapping for {} is not GCS! Found {}, expected {}'
      message.result = deployment_messages.Result.WARNING
      message.text = text.format(bucket_name, content, CNAME)
    else:
      text = 'CNAME for {} -> {}'.format(bucket_name, content, CNAME)
      message.text = text.format(text, content, CNAME)
    return message


class Config(messages.Message):
  bucket = messages.StringField(1)
  access_key = messages.StringField(2)
  access_secret = messages.StringField(3)
  project = messages.StringField(4)
  email = messages.StringField(5)
  key_path = messages.StringField(6)


class GoogleCloudStorageDestination(base.BaseDestination):
  NAME = 'gcs'
  TestCase = TestCase
  Config = Config

  def __str__(self):
    return 'gs://{}'.format(self.config.bucket)

  @property
  def use_interoperable_auth(self):
    return self.config.email is None

  @webapp2.cached_property
  def bucket(self):
    if self.use_interoperable_auth:
      gs_connection = boto.connect_gs(
          self.config.access_key, self.config.access_secret,
          calling_format=connection.OrdinaryCallingFormat())
    else:
      gs_connection = storage.get_connection(
          self.config.project, self.config.email, self.config.key_path)
    return gs_connection.get_bucket(self.config.bucket)

  def prelaunch(self, dry_run=False):
    if dry_run:
      return
    logging.info('Configuring GS bucket: {}'.format(self.config.bucket))
    if self.use_interoperable_auth:
      self.bucket.set_acl('public-read')
      self.bucket.configure_versioning(False)
      self.bucket.configure_website(main_page_suffix='index.html', error_key='<KEY>')
    else:
      acl = self.bucket.get_default_object_acl()
      acl.all().grant_read().revoke_write()
      acl.save()
      self.bucket.configure_website(main_page_suffix='index.html', not_found_page='404.html')

  def write_control_file(self, path, content):
    path = os.path.join(self.control_dir, path.lstrip('/'))
    return self.write_file(path, content, policy='private')

  def read_file(self, path):
{'fraction_non_alphanumeric': 0.11069692058346839, 'fraction_numerical': 0.005510534846029173, 'mean_word_length': 3.9486768243785084, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3866818', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1929, 'n_words': 398}