<commit_before>require 'google/apis/drive_v3'
require 'google/apis/sheets_v4'
require 'googleauth'
require 'googleauth/stores/file_token_store'
require 'fileutils'

OOB_URI = 'urn:ietf:wg:oauth:2.0:oob'.freeze
APPLICATION_NAME = 'Student Insights'.freeze
CREDENTIALS_PATH = 'credentials.json'.freeze
# The file token.yaml stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
TOKEN_PATH = 'token.yaml'.freeze
SCOPE = [Google::Apis::DriveV3::AUTH_DRIVE_READONLY, Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY]

##
# Ensure valid credentials, either by restoring from the saved credentials
# files or intitiating an OAuth2 authorization. If authorization is required,
# the user's default browser will be launched to approve the request.
#
# @return [Google::Auth::UserRefreshCredentials] OAuth2 credentials
def authorize
  client_id = Google::Auth::ClientId.from_file(CREDENTIALS_PATH)
  token_store = Google::Auth::Stores::FileTokenStore.new(file: TOKEN_PATH)
  authorizer = Google::Auth::UserAuthorizer.new(client_id, SCOPE, token_store)
  user_id = 'default'
  credentials = authorizer.get_credentials(user_id)
  if credentials.nil?
    url = authorizer.get_authorization_url(base_url: OOB_URI)
    puts 'Open the following URL in the browser and enter the ' \
         "resulting code after authorization:\n" + url
    code = gets
    credentials = authorizer.get_and_store_credentials_from_code(
      user_id: user_id, code: code, base_url: OOB_URI
    )
  end
  credentials
end

# Initialize the APIs

drive_service = Google::Apis::DriveV3::DriveService.new
drive_service.client_options.application_name = APPLICATION_NAME
drive_service.authorization = authorize

service = Google::Apis::SheetsV4::SheetsService.new
service.client_options.application_name = APPLICATION_NAME
service.authorization = authorize

# Get folder ID

folder_name = "Sheets imports"
folder = drive_service.list_files(q: "name = '#{folder_name}'",
                                    fields: 'files(id, name)')
folder_id = folder.files[0].id

# Get GIDs for all sheets in folder

sheets_info = drive_service.list_files(q: "'#{folder_id}' in parents",
                                  fields: 'files(id, name, properties)')

# response = drive_service.list_files(q: "parents in '1c17xHZhnVzwREt8Q8UKlnzuf9xiaQBxN'",
#                                     fields: 'nextPageToken, files(id, name)')

puts sheets_info.files.size

# sheets_info.files.each do |sheet|
#   puts sheet.properties
# end

puts 'Files:'
puts 'No files found' if sheets_info.files.empty?
sheets_info.files.each do |file|
  puts "#{file.name} (#{file.id}) #{file.properties}"
end

# sheets_info.files.each do |file|
#   dloaded = drive_service.export_file(file.id,
#                             'text/csv',
#                             download_dest: StringIO.new)
#   puts dloaded.string
# end<commit_msg>adding sheets<commit_after>require 'google/apis/drive_v3'
require 'google/apis/sheets_v4'
require 'googleauth'
require 'googleauth/stores/file_token_store'
require 'fileutils'

OOB_URI = 'urn:ietf:wg:oauth:2.0:oob'.freeze
APPLICATION_NAME = 'Student Insights'.freeze
CREDENTIALS_PATH = 'credentials.json'.freeze
# The file token.yaml stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
TOKEN_PATH = 'token.yaml'.freeze
SCOPE = [Google::Apis::DriveV3::AUTH_DRIVE_READONLY, Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY]

##
# Ensure valid credentials, either by restoring from the saved credentials
# files or intitiating an OAuth2 authorization. If authorization is required,
# the user's default browser will be launched to approve the request.
#
# @return [Google::Auth::UserRefreshCredentials] OAuth2 credentials
def authorize
  client_id = Google::Auth::ClientId.from_file(CREDENTIALS_PATH)
  token_store = Google::Auth::Stores::FileTokenStore.new(file: TOKEN_PATH)
  authorizer = Google::Auth::UserAuthorizer.new(client_id, SCOPE, token_store)
  user_id = 'default'
  credentials = authorizer.get_credentials(user_id)
  if credentials.nil?
    url = authorizer.get_authorization_url(base_url: OOB_URI)
    puts 'Open the following URL in the browser and enter the ' \
         "resulting code after authorization:\n" + url
    code = gets
    credentials = authorizer.get_and_store_credentials_from_code(
      user_id: user_id, code: code, base_url: OOB_URI
    )
  end
  credentials
end

# Initialize the APIs

drive_service = Google::Apis::DriveV3::DriveService.new
drive_service.client_options.application_name = APPLICATION_NAME
drive_service.authorization = authorize

sheet_service = Google::Apis::SheetsV4::SheetsService.new
sheet_service.client_options.application_name = APPLICATION_NAME
sheet_service.authorization = authorize

# Get folder ID

folder_name = "Sheets imports"
folder = drive_service.list_files(q: "name = '#{folder_name}'",
                                    fields: 'files(id, name)')
folder_id = folder.files[0].id

# Get GIDs for all sheets in folder

sheets_info = drive_service.list_files(q: "'#{folder_id}' in parents",
                                  fields: 'files(id, name)')

sheets_info.files.each do |sheet|
  sheet = sheet_service.get_spreadsheet(sheet.id)
  puts sheet
end

puts 'Files:'
puts 'No files found' if sheets_info.files.empty?
sheets_info.files.each do |file|
  puts "#{file.name} (#{file.id}) #{file.properties}"
end

# sheets_info.files.each do |file|
#   dloaded = drive_service.export_file(file.id,
#                             'text/csv',
#                             download_dest: StringIO.new)
#   puts dloaded.string
# end
{'fraction_non_alphanumeric': 0.10216556060076842, 'fraction_numerical': 0.004889975550122249, 'mean_word_length': 4.3175487465181055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3509728', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1814, 'n_words': 428}