<commit_before># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""The main module that initiates scanning of GCP resources.

"""

import argparse
import json
import logging
import os
import sys
from typing import List, Tuple, Dict, Optional

from . import crawl
from . import credsdb
from google.cloud import container_v1
from google.cloud import iam_credentials
from google.cloud.iam_credentials_v1.services.iam_credentials.client import IAMCredentialsClient
from googleapiclient import discovery
from httplib2 import Credentials
from .models import SpiderContext


def is_set(config, config_setting):
  if config is None:
    return True
  obj = config.get(config_setting, {})
  return obj.get('fetch', False)


def crawl_loop(initial_sa_tuples: List[Tuple[str, Credentials, List[str]]],
               out_dir: str,
               scan_config: Dict,
               target_project: Optional[str] = None,
               force_projects: Optional[str] = None):
  """The main loop function to crawl GCP resources.

  Args:
    initial_sa_tuples: [(sa_name, sa_object, chain_so_far)]
    out_dir: directory to save results
    target_project: project name to scan
    force_projects: a list of projects to force scan
  """

  context = SpiderContext(initial_sa_tuples)
  # Main loop
  processed_sas = set()
  while not context.service_account_queue.empty():
    # Get a new candidate service account / token
    sa_name, credentials, chain_so_far = context.service_account_queue.get()
    if sa_name in processed_sas:
      continue

    # Don't process this service account again
    processed_sas.add(sa_name)
    print('>> current service account: {}'.format(sa_name))
    sa_results = crawl.infinite_defaultdict()
    # Log the chain we used to get here (even if we have no privs)
    sa_results['service_account_chain'] = chain_so_far
    sa_results['current_service_account'] = sa_name

    project_list = crawl.get_project_list(credentials)
    if len(project_list) <= 0:
      print('Unable to list projects accessible from service account')

    if force_projects:
      for force_project_id in force_projects:
        res = crawl.fetch_project_info(force_project_id, credentials)
        if res:
          project_list.append(res)

    # Enumerate projects accessible by SA
    for project in project_list:
      if target_project and target_project not in project['projectId']:
        continue

      project_id = project['projectId']
      project_number = project['projectNumber']
      print('Inspecting project {}'.format(project_id))
      project_result = sa_results['projects'][project_id]

      project_result['project_info'] = project

      if is_set(scan_config, 'iam_policy'):
        # Get IAM policy
        iam_client = iam_client_for_credentials(credentials)
        iam_policy = crawl.get_iam_policy(project_id, credentials)
        project_result['iam_policy'] = iam_policy

      if is_set(scan_config, 'service_accounts'):
        # Get service accounts
        project_service_accounts = crawl.get_service_accounts(
            project_number, credentials)
        project_result['service_accounts'] = project_service_accounts
<commit_msg>Fix issue with forced project scanning<commit_after># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""The main module that initiates scanning of GCP resources.

"""

import argparse
import json
import logging
import os
import sys
from typing import List, Tuple, Dict, Optional

from . import crawl
from . import credsdb
from google.cloud import container_v1
from google.cloud import iam_credentials
from google.cloud.iam_credentials_v1.services.iam_credentials.client import IAMCredentialsClient
from googleapiclient import discovery
from httplib2 import Credentials
from .models import SpiderContext


def is_set(config, config_setting):
  if config is None:
    return True
  obj = config.get(config_setting, {})
  return obj.get('fetch', False)


def crawl_loop(initial_sa_tuples: List[Tuple[str, Credentials, List[str]]],
               out_dir: str,
               scan_config: Dict,
               target_project: Optional[str] = None,
               force_projects: Optional[str] = None):
  """The main loop function to crawl GCP resources.

  Args:
    initial_sa_tuples: [(sa_name, sa_object, chain_so_far)]
    out_dir: directory to save results
    target_project: project name to scan
    force_projects: a list of projects to force scan
  """

  context = SpiderContext(initial_sa_tuples)
  # Main loop
  processed_sas = set()
  while not context.service_account_queue.empty():
    # Get a new candidate service account / token
    sa_name, credentials, chain_so_far = context.service_account_queue.get()
    if sa_name in processed_sas:
      continue

    # Don't process this service account again
    processed_sas.add(sa_name)
    print('>> current service account: {}'.format(sa_name))
    sa_results = crawl.infinite_defaultdict()
    # Log the chain we used to get here (even if we have no privs)
    sa_results['service_account_chain'] = chain_so_far
    sa_results['current_service_account'] = sa_name

    project_list = crawl.get_project_list(credentials)
    if len(project_list) <= 0:
      print('Unable to list projects accessible from service account')

    if force_projects:
      for force_project_id in force_projects:
        res = crawl.fetch_project_info(force_project_id, credentials)
        if res:
          project_list.append(res)
        else:
          # force object creation
          project_list.append({'projectId': force_project_id, 'projectNumber': 'N/A'})

    # Enumerate projects accessible by SA
    for project in project_list:
      if target_project and target_project not in project['projectId']:
        continue

      project_id = project['projectId']
      project_number = project['projectNumber']
      print('Inspecting project {}'.format(project_id))
      project_result = sa_results['projects'][project_id]

      project_result['project_info'] = project

      if is_set(scan_config, 'iam_policy'):
        # Get IAM policy
        iam_client = iam_client_for_credentials(credentials)
        iam_policy = crawl.get_iam_policy(project_id, credentials)
        project_result['iam_policy'] = iam_policy

      if is_set(scan_config, 'service_accounts'):
        # Get service accounts
        project_service_accounts = crawl.get_service_accounts(
            project_number, credentials)
        project_result['service_accounts'] = project_service_accounts

{'fraction_non_alphanumeric': 0.0755490870600688, 'fraction_numerical': 0.0031754432389521038, 'mean_word_length': 3.763074984247007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3365806', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2093, 'n_words': 681}