<commit_before><commit_msg>First commit of the ripple.rb file. This can connect to Dropbox, and report information about files, but does not download anything yet.<commit_after>#!/usr/bin/ruby
require 'fileutils'
require 'yaml'
require "rubygems"
require 'dropbox_sdk'

conf = {:destinationDir => './test',
        :cleanDestiation => false}

# Load configuration.
if File.exists?('ripple.yaml')
  conf = YAML::load(File.read('ripple.yaml'))
end

#Load Dropbox API login from file, if applicable.
if File.exists?('keys.yaml')
  login = YAML::load(File.read('keys.yaml'))
else
  puts "Please enter your Dropbox API key"
  login['key'] = gets.chomp!
  puts "Please enter your Dropbox API secret"
  login['secret'] = gets.chomp!
end

if !conf.has_key?(:destinationDir)
  conf[:destinationDir] = './test'
end

destDir = conf[:destinationDir]

if conf.has_key?(:cleanDestination)
  if conf[:cleanDestination]
    files = Dir.glob(destDir + "**")

    if files.empty?
      FileUtils.rm(files)
    end
  end
end

session = nil
ACCESS_TYPE = :app_folder

if File.exists?('dropbox_session.yaml')
  deser = File.open('dropbox_session.yaml').read
  session = DropboxSession.deserialize(deser)
else
  session = DropboxSession.new(login[:key], login[:secret])
  session.get_request_token
  authorize_url = session.get_authorize_url
  puts "Visit #{authorize_url} to log in to Dropbox. Hit enter when you have done this."
  gets
  session.get_access_token
end

if session == nil
  exit
end
client = DropboxClient.new(session, ACCESS_TYPE)

#Here we need to actually sync newest files.
puts client.metadata('/')

File.open('dropbox_session.yaml', 'w') do|file|
  file.puts session.serialize()
end

File.open('ripple.yaml', 'w') do |file|
  file.puts conf.to_yaml
end
{'fraction_non_alphanumeric': 0.1278538812785388, 'fraction_numerical': 0.0, 'mean_word_length': 4.65483870967742, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5064332', 'n_tokens_mistral': 589, 'n_tokens_neox': 583, 'n_words': 159}