<commit_before># Copyright 2016 Google, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in write, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def list_bucket_contents project_id:, bucket_name:
  # [START list_bucket_contents]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name

  bucket.files.each do |file|
    puts file.name
  end
  # [END list_bucket_contents]
end

def list_bucket_contents_with_prefix project_id:, bucket_name:, prefix:
  # [START list_bucket_contents_with_prefix]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # prefix      = "Filter results to files whose names begin with this prefix"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  files   = bucket.files prefix: prefix

  files.each do |file|
    puts file.name
  end
  # [END list_bucket_contents_with_prefix]
end

def generate_encryption_key_base64
  require "base64"
  require "openssl"

  encryption_key  = OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key
  encoded_enc_key = Base64.encode64 encryption_key

  puts "Sample encryption key: #{encoded_enc_key}"
end

def upload_file project_id:, bucket_name:, local_file_path:,
                                           storage_file_path:
  # [START upload_file]
  # project_id        = "Your Google Cloud project ID"
  # bucket_name       = "Your Google Cloud Storage bucket name"
  # local_file_path   = "Path to local file to upload"
  # storage_file_path = "Path to store the file in Google Cloud Storage"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name

  file = bucket.create_file local_file_path, storage_file_path

  puts "Uploaded #{file.name}"
  # [END upload_file]
end

def upload_encrypted_file project_id:, bucket_name:, local_file_path:,
                          storage_file_path: nil, encryption_key:
  # [START upload_encryption_file]
  # project_id        = "Your Google Cloud project ID"
  # bucket_name       = "Your Google Cloud Storage bucket name"
  # local_file_path   = "Path to local file to upload"
  # storage_file_path = "Path to store the file in Google Cloud Storage"
  # encryption_key    = "AES-256 encryption key"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id

  bucket = storage.bucket bucket_name

  file = bucket.create_file local_file_path, storage_file_path,
                            encryption_key: encryption_key

  puts "Uploaded #{file.name} with encryption key"
end

def download_file project_id:, bucket_name:, file_name:, local_path:
  # [START download_file]
  # project_id     = "Your Google Cloud project ID"
  # bucket_name    = "Your Google Cloud Storage bucket name"
  # file_name      = "Name of file in Google Cloud Storage to download locally"
  # local_path     = "Path to local file to save"
  # encryption_key = "Base64 encoded AES-256 encryption key"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  file    = bucket.file file_name

  file.download local_path

  puts "Downloaded #{file.name}"
  # [END download_file]
end

def download_encrypted_file project_id:, bucket_name:, storage_file_path:,
                            local_file_path:, encryption_key:
  # [START download_file]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # file_name   = "Name of file in Google Cloud Storage to download locally"
  # local_path  = "Path to local file to save"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id

  bucket = storage.bucket bucket_name

  file = bucket.file storage_file_path, encryption_key: encryption_key
  file.download local_file_path, encryption_key: encryption_key

  puts "Downloaded encrypted #{file.name}"
end

def delete_file project_id:, bucket_name:, file_name:
  # [START delete_file]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # file_name   = "Name of file in Google Cloud Storage to delete"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  file    = bucket.file file_name

  file.delete

  puts "Deleted #{file.name}"
  # [END delete_file]
end
<commit_msg>Fixing comment and moving require statements<commit_after># Copyright 2016 Google, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in write, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

require "base64"
require "openssl"

def list_bucket_contents project_id:, bucket_name:
  # [START list_bucket_contents]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name

  bucket.files.each do |file|
    puts file.name
  end
  # [END list_bucket_contents]
end

def list_bucket_contents_with_prefix project_id:, bucket_name:, prefix:
  # [START list_bucket_contents_with_prefix]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # prefix      = "Filter results to files whose names begin with this prefix"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  files   = bucket.files prefix: prefix

  files.each do |file|
    puts file.name
  end
  # [END list_bucket_contents_with_prefix]
end

def generate_encryption_key_base64
  encryption_key  = OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key
  encoded_enc_key = Base64.encode64 encryption_key

  puts "Sample encryption key: #{encoded_enc_key}"
end

def upload_file project_id:, bucket_name:, local_file_path:,
                                           storage_file_path:
  # [START upload_file]
  # project_id        = "Your Google Cloud project ID"
  # bucket_name       = "Your Google Cloud Storage bucket name"
  # local_file_path   = "Path to local file to upload"
  # storage_file_path = "Path to store the file in Google Cloud Storage"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name

  file = bucket.create_file local_file_path, storage_file_path

  puts "Uploaded #{file.name}"
  # [END upload_file]
end

def upload_encrypted_file project_id:, bucket_name:, local_file_path:,
                          storage_file_path: nil, encryption_key:
  # [START upload_encryption_file]
  # project_id        = "Your Google Cloud project ID"
  # bucket_name       = "Your Google Cloud Storage bucket name"
  # local_file_path   = "Path to local file to upload"
  # storage_file_path = "Path to store the file in Google Cloud Storage"
  # encryption_key    = "AES-256 encryption key"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id

  bucket = storage.bucket bucket_name

  file = bucket.create_file local_file_path, storage_file_path,
                            encryption_key: encryption_key

  puts "Uploaded #{file.name} with encryption key"
end

def download_file project_id:, bucket_name:, file_name:, local_path:
  # [START download_file]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # file_name   = "Name of file in Google Cloud Storage to download locally"
  # local_path  = "Path to local file to save"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  file    = bucket.file file_name

  file.download local_path

  puts "Downloaded #{file.name}"
  # [END download_file]
end

def download_encrypted_file project_id:, bucket_name:, storage_file_path:,
                            local_file_path:, encryption_key:
  # [START download_file]
  # project_id     = "Your Google Cloud project ID"
  # bucket_name    = "Your Google Cloud Storage bucket name"
  # file_name      = "Name of file in Google Cloud Storage to download locally"
  # local_path     = "Path to local file to save"
  # encryption_key = "AES-256 encryption key"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id

  bucket = storage.bucket bucket_name

  file = bucket.file storage_file_path, encryption_key: encryption_key
  file.download local_file_path, encryption_key: encryption_key

  puts "Downloaded encrypted #{file.name}"
end

def delete_file project_id:, bucket_name:, file_name:
  # [START delete_file]
  # project_id  = "Your Google Cloud project ID"
  # bucket_name = "Your Google Cloud Storage bucket name"
  # file_name   = "Name of file in Google Cloud Storage to delete"

  require "google/cloud/storage"

  storage = Google::Cloud::Storage.new project: project_id
  bucket  = storage.bucket bucket_name
  file    = bucket.file file_name

  file.delete

  puts "Deleted #{file.name}"
  # [END delete_file]
end

{'fraction_non_alphanumeric': 0.07891156462585033, 'fraction_numerical': 0.005053449951409135, 'mean_word_length': 3.8043884220354807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5956991', 'n_tokens_mistral': 3081, 'n_tokens_neox': 3002, 'n_words': 983}