<commit_before>require "fog/google"
require "json"
google_creds = JSON.load(File.open('.google_creds.json'))
goog_config = {
  google_project: google_creds['project_id'],
  google_client_email: google_creds['client_email'],
  google_json_key_location: '.google_creds.json'
}
goog = Fog::Compute::Google.new(config=goog_config)

web = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.type = app')

memcached = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.role = memcached')

elastic = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.role = elasticsearch')

set :capenv, ->(env) {
  Dotenv.load('.env.prod').each {|k,v| env.add k,v}
  # we're using cloud memorystore for redis and fog doesn't support querying it so hard-code it
  env.add 'REDISCLOUD_URL', "redis://10.0.1.4:6379/0"
  env.add 'MEMCACHIER_SERVERS', memcached.map {|mem| mem.network_interfaces[0][:network_ip] + ':11211'}.join(',')
  env.add 'ELASTICSEARCH_CLUSTER_URL', elastic.map {|es| es.network_interfaces[0][:network_ip] + ':9200'}.join(',')
}

web.each do |s|
  server s.network_interfaces[0][:access_configs][0][:nat_ip], user: 'deploy', roles: [s.labels.fetch(:type, ''), s.labels.fetch(:role, '')]
end
<commit_msg>Add capistrano tasks around starting to run on k8s<commit_after>require "fog/google"
require "json"
google_creds = JSON.load(File.open('.google_creds.json'))
goog_config = {
  google_project: google_creds['project_id'],
  google_client_email: google_creds['client_email'],
  google_json_key_location: '.google_creds.json'
}
goog = Fog::Compute::Google.new(config=goog_config)

web = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.type = app')

memcached = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.role = memcached')

elastic = goog.servers.all(filter: 'labels.project = libraries AND labels.environment = production AND labels.role = elasticsearch')

set :capenv, ->(env) {
  Dotenv.load('.env.prod').each {|k,v| env.add k,v}
  # we're using cloud memorystore for redis and fog doesn't support querying it so hard-code it
  env.add 'REDISCLOUD_URL', "redis://10.0.1.4:6379/0"
  env.add 'MEMCACHIER_SERVERS', memcached.map {|mem| mem.network_interfaces[0][:network_ip] + ':11211'}.join(',')
  env.add 'ELASTICSEARCH_CLUSTER_URL', elastic.map {|es| es.network_interfaces[0][:network_ip] + ':9200'}.join(',')
}

web.each do |s|
  next unless s.status == "RUNNING"
  server s.network_interfaces[0][:access_configs][0][:nat_ip], user: 'deploy', roles: [s.labels.fetch(:type, ''), s.labels.fetch(:role, '')]
end

namespace :deploy do
  task :tag_docker_live do
    revision = `git show-ref origin/master`.split.first
    system "gcloud --quiet container images add-tag gcr.io/#{ENV['GOOGLE_PROJECT']}/libraries.io:#{revision} gcr.io/#{ENV['GOOGLE_PROJECT']}/libraries.io:latest"
  end
  after :published , :tag_docker_live

  task :update_k8s do
    revision = `git show-ref origin/master`.split.first
    system "kubectl set image deployment/libraries-sidekiq-worker-deploy libraries-sidekiq-worker=gcr.io/#{ENV['GOOGLE_PROJECT']}/libraries.io:#{revision}"
  end
  before :publishing, :update_k8s

  task :ensure_container_built do
    checks = 0
    revision = `git show-ref origin/master`.split.first
    while checks < 10 && !system("gcloud container images describe gcr.io/#{ENV['GOOGLE_PROJECT']}/libraries.io:#{revision}") do
      checks += 1
      print "Waiting for revision #{revision} to be built on Google Container Registry..."
      sleep(60)
    end
  end
  after :starting, :ensure_container_built
end

{'fraction_non_alphanumeric': 0.15145888594164456, 'fraction_numerical': 0.014588859416445624, 'mean_word_length': 5.327181208053691, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5257070', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1297, 'n_words': 276}