<commit_before>common_sa_org_roles = [
  "roles/dns.admin",
  "roles/iam.organizationRoleViewer",
  "roles/iam.serviceAccountAdmin",
  "roles/iam.serviceAccountKeyAdmin",
  "roles/resourcemanager.projectIamAdmin",
  "roles/resourcemanager.projectCreator",
  "roles/serviceusage.serviceUsageAdmin",
  "roles/storage.admin",
  "roles/viewer",
]

cloud_admin_org_roles = [
  "roles/billing.admin",
  "roles/cloudsupport.admin",
  "roles/orgpolicy.policyAdmin",
  "roles/resourcemanager.organizationAdmin",
  "roles/securitycenter.viewer",
  "roles/viewer",
]

project_services = [
  "cloudresourcemanager.googleapis.com",
  "cloudbilling.googleapis.com",
  "containeranalysis.googleapis.com",
  "containerregistry.googleapis.com",
  "iam.googleapis.com",
  "dns.googleapis.com",
  "compute.googleapis.com",
  "securitycenter.googleapis.com",
]

task :refresh_common_infra, [:project_type] => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do | taskname, args|

  next if args[:project_type] == "common"

  # The common infrastructure might create the DNS resources, in order to avoid
  # conflicts at the creation resources we need to import the already created
  # DNS zone.

  dns_zone_found = %x{
    #{@exekube_cmd} sh -c " \
    terragrunt state show module.gke_network.google_dns_managed_zone.dns_zones \
    --terragrunt-working-dir /project/live/#{@env}/infra/network 2>/dev/null \
  "}

  if dns_zone_found.empty?
    # The DNS zone is not in the TF state file, we need to add it
    puts "DNS zone #{ENV["TF_VAR_domain_name"].tr('.','-')} not found in TF state, importing..."
    sh "#{@exekube_cmd} sh -c ' \
      terragrunt import module.gke_network.google_dns_managed_zone.dns_zones #{ENV["TF_VAR_domain_name"].tr('.','-')} \
      --terragrunt-working-dir /project/live/#{@env}/infra/network \
    '"
  else
    # The DNS zone is in the TF state file, we will refresh it just in case
    puts "DNS zone #{ENV["TF_VAR_domain_name"].tr('.','-')} found in TF state, refreshing..."
    sh "#{@exekube_cmd} sh -c ' \
      terragrunt refresh \
      --terragrunt-working-dir /project/live/#{@env}/infra/network \
    '"
  end
end

task :apply_common_infra => [@gcp_creds_file] do
  # Steps to initialize GCP with a minimum set of resources to allow Terraform
  # create the rest of the infrastructure.
  # These steps are the same found in this tutorial:
  # https://cloud.google.com/community/tutorials/managing-gcp-projects-with-terraform
  #
  # Due to the needed permissions for creating the following resources, only an
  # administrator of the organization can run this task.

  projects_list = %x{
    #{@exekube_cmd} gcloud projects list --format='json' --filter='name:#{ENV["TF_VAR_project_id"]}'
  }
  projects_list = JSON.parse(projects_list)
  if projects_list.empty?
    puts "#{ENV["TF_VAR_project_id"]} not found, I'll create a new one"
    sh "#{@exekube_cmd} gcloud projects create #{ENV["TF_VAR_project_id"]} \
      --organization #{ENV["ORGANIZATION_ID"]} \
      --set-as-default"
  end

  Rake::Task[:configure_current_project].invoke

  sh "#{@exekube_cmd} gcloud beta billing projects link #{ENV["TF_VAR_project_id"]} \
    --billing-account #{ENV["BILLING_ID"]}"

  sa_list = %x{
    #{@exekube_cmd} gcloud iam service-accounts list --format='json' \
    --filter='email:<EMAIL>@#{ENV["TF_VAR_project_id"]}.<EMAIL>'
  }
  sa_list = JSON.parse(sa_list)
  if sa_list.empty?
    sh "#{@exekube_cmd} gcloud iam service-accounts create projectowner --display-name 'CI account'"
  end

  Rake::Task[:configure_serviceaccount].invoke
  Rake::Task[:set_org_perms].invoke

  services_list = %x{
    #{@exekube_cmd} gcloud services list --format='json'
  }
  services_list = JSON.parse(services_list)

  project_services.each do |service|
     sh "#{@exekube_cmd} gcloud services enable #{service}" unless services_list.any? { |s| s['serviceName'] == service }
  end

  tfstate_bucket = %x{
    #{@exekube_cmd} gsutil ls | grep 'gs://#{ENV["TF_VAR_project_id"]}-tfstate'
  }
  unless tfstate_bucket
    sh "#{@exekube_cmd} gsutil mb gs://#{ENV["TF_VAR_project_id"]}-tfstate"
    sh "#{@exekube_cmd} gsutil versioning set on gs://#{ENV["TF_VAR_project_id"]}-tfstate"
  end
end

task :set_org_perms => [@gcp_creds_file] do
  # Remove org-level roles from individual user accounts if found,
  # to forcefully revoke all org-level permissions granted by :grant_org_admin
  user_roles = %x{
    #{@exekube_cmd} gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
    --filter bindings.members:user:* \
    --flatten="bindings[].members" \
    --format json | jq -r ".[].bindings.members"
  }.split.uniq.each do |user|
    existing_user_roles = %x{
      #{@exekube_cmd} gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
      --filter bindings.members:#{user} \
      --flatten="bindings[].members" \
      --format json | jq -r ".[].bindings.role"
    }.split.each do |role|
      sh "#{@exekube_cmd} gcloud organizations remove-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{user} \
        --role #{role}"
    end
  end
  # Enforce org-level roles for cloud-admin, common SA, and Eugene's account
  members = {
    "group:<EMAIL>" => cloud_admin_org_roles,
    "serviceAccount:<EMAIL>@#{ENV["TF_VAR_project_id"]}.<EMAIL>" => common_sa_org_roles,
    "user:<EMAIL>" => ["roles/billing.admin"]
  }
  members.each do |member, expected_roles|
    existing_roles = %x{
      #{@exekube_cmd} gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
      --filter bindings.members:#{member} \
      --flatten="bindings[].members" \
      --format json | jq -r ".[].bindings.role"
    }.split

    roles_to_add = expected_roles - existing_roles
    roles_to_remove = existing_roles - expected_roles

    roles_to_add.each do |role|
      sh "#{@exekube_cmd} gcloud organizations add-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{member} \
        --role #{role}"
    end
    roles_to_remove.each do |role|
      sh "#{@exekube_cmd} gcloud organizations remove-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{member} \
        --role #{role}"
    end
  end
  # The billing account is owned by the organization 247149361674
  # (raisingthefloor.org), that means that the permissions are inherited from
  # such organization. All the SA that need a billing permission must be in this
  # organization IAM settings.
  # Go to the https://console.cloud.google.com/billing/ to see the permissions
  # granted to which SA for using billing services.
  sh "#{@exekube_cmd} gcloud organizations add-iam-policy-binding 247149361674 \
    --member serviceAccount:<EMAIL>@#{ENV["TF_VAR_project_id"]}.iam.<EMAIL>account.com \
    --role roles/billing.user"
end

task :plan_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} plan-all live/#{@env}/infra 2> /dev/null"
end

task :apply_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} up live/#{@env}/infra"
end

task :destroy_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} down live/#{@env}/infra"
end<commit_msg>exekube_cmd is not needed<commit_after>common_sa_org_roles = [
  "roles/dns.admin",
  "roles/iam.organizationRoleViewer",
  "roles/iam.serviceAccountAdmin",
  "roles/iam.serviceAccountKeyAdmin",
  "roles/resourcemanager.projectIamAdmin",
  "roles/resourcemanager.projectCreator",
  "roles/serviceusage.serviceUsageAdmin",
  "roles/storage.admin",
  "roles/viewer",
]

cloud_admin_org_roles = [
  "roles/billing.admin",
  "roles/cloudsupport.admin",
  "roles/orgpolicy.policyAdmin",
  "roles/resourcemanager.organizationAdmin",
  "roles/securitycenter.viewer",
  "roles/viewer",
]

project_services = [
  "cloudresourcemanager.googleapis.com",
  "cloudbilling.googleapis.com",
  "containeranalysis.googleapis.com",
  "containerregistry.googleapis.com",
  "iam.googleapis.com",
  "dns.googleapis.com",
  "compute.googleapis.com",
  "securitycenter.googleapis.com",
]

task :refresh_common_infra, [:project_type] => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do | taskname, args|

  next if args[:project_type] == "common"

  # The common infrastructure might create the DNS resources, in order to avoid
  # conflicts at the creation resources we need to import the already created
  # DNS zone.

  dns_zone_found = %x{
    sh -c " \
    terragrunt state show module.gke_network.google_dns_managed_zone.dns_zones \
    --terragrunt-working-dir /project/live/#{@env}/infra/network 2>/dev/null \
  "}

  if dns_zone_found.empty?
    # The DNS zone is not in the TF state file, we need to add it
    puts "DNS zone #{ENV["TF_VAR_domain_name"].tr('.','-')} not found in TF state, importing..."
    sh "sh -c ' \
      terragrunt import module.gke_network.google_dns_managed_zone.dns_zones #{ENV["TF_VAR_domain_name"].tr('.','-')} \
      --terragrunt-working-dir /project/live/#{@env}/infra/network \
    '"
  else
    # The DNS zone is in the TF state file, we will refresh it just in case
    puts "DNS zone #{ENV["TF_VAR_domain_name"].tr('.','-')} found in TF state, refreshing..."
    sh "sh -c ' \
      terragrunt refresh \
      --terragrunt-working-dir /project/live/#{@env}/infra/network \
    '"
  end
end

task :apply_common_infra => [@gcp_creds_file] do
  # Steps to initialize GCP with a minimum set of resources to allow Terraform
  # create the rest of the infrastructure.
  # These steps are the same found in this tutorial:
  # https://cloud.google.com/community/tutorials/managing-gcp-projects-with-terraform
  #
  # Due to the needed permissions for creating the following resources, only an
  # administrator of the organization can run this task.

  projects_list = %x{
    gcloud projects list --format='json' --filter='name:#{ENV["TF_VAR_project_id"]}'
  }
  projects_list = JSON.parse(projects_list)
  if projects_list.empty?
    puts "#{ENV["TF_VAR_project_id"]} not found, I'll create a new one"
    sh "gcloud projects create #{ENV["TF_VAR_project_id"]} \
      --organization #{ENV["ORGANIZATION_ID"]} \
      --set-as-default"
  end

  Rake::Task[:configure_current_project].invoke

  sh "gcloud beta billing projects link #{ENV["TF_VAR_project_id"]} \
    --billing-account #{ENV["BILLING_ID"]}"

  sa_list = %x{
    gcloud iam service-accounts list --format='json' \
    --filter='email:projectowner@#{ENV["TF_VAR_project_id"]}.iam.gserviceaccount.com'
  }
  sa_list = JSON.parse(sa_list)
  if sa_list.empty?
    sh "gcloud iam service-accounts create projectowner --display-name 'CI account'"
  end

  Rake::Task[:configure_serviceaccount].invoke
  Rake::Task[:set_org_perms].invoke

  services_list = %x{
    gcloud services list --format='json'
  }
  services_list = JSON.parse(services_list)

  project_services.each do |service|
     sh "gcloud services enable #{service}" unless services_list.any? { |s| s['serviceName'] == service }
  end

  tfstate_bucket = %x{
    gsutil ls | grep 'gs://#{ENV["TF_VAR_project_id"]}-tfstate'
  }
  unless tfstate_bucket
    sh "gsutil mb gs://#{ENV["TF_VAR_project_id"]}-tfstate"
    sh "gsutil versioning set on gs://#{ENV["TF_VAR_project_id"]}-tfstate"
  end
end

task :set_org_perms => [@gcp_creds_file] do
  # Remove org-level roles from individual user accounts if found,
  # to forcefully revoke all org-level permissions granted by :grant_org_admin
  user_roles = %x{
    gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
    --filter bindings.members:user:* \
    --flatten="bindings[].members" \
    --format json | jq -r ".[].bindings.members"
  }.split.uniq.each do |user|
    existing_user_roles = %x{
      gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
      --filter bindings.members:#{user} \
      --flatten="bindings[].members" \
      --format json | jq -r ".[].bindings.role"
    }.split.each do |role|
      sh "gcloud organizations remove-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{user} \
        --role #{role}"
    end
  end
  # Enforce org-level roles for cloud-admin, common SA, and Eugene's account
  members = {
    "group:<EMAIL>" => cloud_admin_org_roles,
    "serviceAccount:<EMAIL>@#{ENV["TF_VAR_project_id"]}.<EMAIL>" => common_sa_org_roles,
    "user:<EMAIL>" => ["roles/billing.admin"]
  }
  members.each do |member, expected_roles|
    existing_roles = %x{
      gcloud organizations get-iam-policy #{ENV["ORGANIZATION_ID"]} \
      --filter bindings.members:#{member} \
      --flatten="bindings[].members" \
      --format json | jq -r ".[].bindings.role"
    }.split

    roles_to_add = expected_roles - existing_roles
    roles_to_remove = existing_roles - expected_roles

    roles_to_add.each do |role|
      sh "gcloud organizations add-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{member} \
        --role #{role}"
    end
    roles_to_remove.each do |role|
      sh "gcloud organizations remove-iam-policy-binding #{ENV["ORGANIZATION_ID"]} \
        --member #{member} \
        --role #{role}"
    end
  end
  # The billing account is owned by the organization 247149361674
  # (raisingthefloor.org), that means that the permissions are inherited from
  # such organization. All the SA that need a billing permission must be in this
  # organization IAM settings.
  # Go to the https://console.cloud.google.com/billing/ to see the permissions
  # granted to which SA for using billing services.
  sh "gcloud organizations add-iam-policy-binding 247149361674 \
    --member serviceAccount:<EMAIL>@#{ENV["TF_VAR_project_id"]}.<EMAIL> \
    --role roles/billing.user"
end

task :plan_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} plan-all live/#{@env}/infra 2> /dev/null"
end

task :apply_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} up live/#{@env}/infra"
end

task :destroy_infra => [@gcp_creds_file, @app_default_creds_file, :configure_extra_tf_vars] do
  sh "#{@exekube_cmd} down live/#{@env}/infra"
end
{'fraction_non_alphanumeric': 0.13946922642574816, 'fraction_numerical': 0.0036702428006775835, 'mean_word_length': 3.8457592339261284, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 39, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1053917', 'n_tokens_mistral': 5021, 'n_tokens_neox': 4876, 'n_words': 1096}