<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""list instances from AWS, and GCP."""
import sys

from argparse import ArgumentParser
import boto3
import botocore
from botocore.exceptions import ClientError
from collections import namedtuple
from concurrent.futures import ThreadPoolExecutor
from dateutil.parser import parse as parse_date
from functools import partial
import kubernetes.client.rest
import kubernetes.config.config_exception
from pytz import UTC
import re
from texttable import Texttable

try:
    from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
    import google.auth.exceptions  # pylint: disable=g-import-not-at-top
    from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
    from googleapiclient import errors  # pylint: disable=g-import-not-at-top
except ImportError:
    print "Warning, google-api-python-client or google-auth not installed, can't connect to GCP instances."

try:
    from kubernetes import client, config  # pylint: disable=g-import-not-at-top,g-multiple-import
except ImportError:
    print "Warning, kubernetes not installed, not getting pods"

AUTOSCALING_GROUP_TAG = u'aws:autoscaling:groupName'
AWS_REGION_LIST = ['us-east-1', 'eu-west-1', 'us-west-2',
                   'sa-east-1', 'ap-southeast-1', 'eu-central-1']
GCP_REGION_LIST = ['us-east1', 'asia-east1', 'asia-northeast1',
                   'asia-southeast1', 'europe-west1', 'us-central1', 'us-west1']
INSTANCE_FILEDS = ['cloud', 'created', 'name', 'region', 'id', 'state', 'ip_address', 'type', 'autoscaling_group',
                   'iam_or_service_account', 'public_dns_name', 'private_ip_address', 'project', 'zone', 'security_groups',
                   'tags', 'vpc_id', 'reservation_type']
DEFAULT_SHOWN_FIELDS = ['project', 'name', 'zone', 'id', 'state', 'ip_address', 'type', 'created',
                        'autoscaling_group', 'iam_or_service_account', 'private_ip_address']
GCP_INSTANCE_TYPE_DICT = {'standard': 'std', 'highmem': 'mem', 'n1-': '', 'highcpu': 'cpu', 'custom': 'ctm'}
ALIGN_OPTIONS = ['l', 'c', 'r']
Instance = namedtuple('Instance', INSTANCE_FILEDS)
SUPPORTED_CLOUDS = ('gcp', 'aws', 'k8s')


def pretify_field(field):
    return ' '.join([word[0].upper() + word[1:] for word in field.split('_')])


def aws_get_instances_by_name(region, name, raw=True):
    """Get all instances in a region matching a name (with wildcards)."""
    return _aws_get_instance_by_tag(region, name, 'tag:Name', raw)


def aws_get_instances_by_autoscaling_group_in_region(region, name, raw=True):
    return _aws_get_instance_by_tag(region, name, 'tag:aws:autoscaling:groupName', raw)


def datetime_to_str(date):
    return date.astimezone(UTC).isoformat().split('+')[0].split('.')[0].replace('T', ' ')


def _aws_instance_from_dict(region, instance, raw):
    tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])}
    return Instance(cloud='aws',  # pylint: disable=expression-not-assigned
                    zone=instance['Placement']['AvailabilityZone'],
                    region=region,
                    id=instance['InstanceId'],
                    name=tags.get('Name', 'Empty'),
                    state=instance['State']['Name'],
                    type=instance['InstanceType'] if raw else instance['InstanceType'].replace('xlarge', 'xl'),
                    autoscaling_group=tags.get(AUTOSCALING_GROUP_TAG, None),
                    public_dns_name=instance['PublicDnsName'],
                    ip_address=instance.get('PublicIpAddress'),
                    iam_or_service_account=instance['IamInstanceProfile'][
                        'Arn'].split('/')[1] if instance.get('IamInstanceProfile') else None,
                    private_ip_address=instance.get('PrivateIpAddress'),
                    project='aws',
                    security_groups=[group['GroupName'] for group in instance['SecurityGroups']],
                    tags=tags,
                    created=datetime_to_str(instance['LaunchTime']),
                    reservation_type=instance.get('InstanceLifecycle', 'on_demand'),
                    vpc_id=instance.get('VpcId'))


def _aws_get_instance_by_tag(region, name, tag, raw):
    """Get all instances matching a tag."""
    client = boto3.session.Session().client('ec2', region)
    matching_reservations = client.describe_instances(Filters=[{'Name': tag, 'Values': [name]}]).get('Reservations', [])
    instances = []
    [[instances.append(_aws_instance_from_dict(region, instance, raw))  # pylint: disable=expression-not-assigned
      for instance in reservation.get('Instances')] for reservation in matching_reservations if reservation]
    return instances


def gcp_filter_projects_instances(projects, filters, raw=True, credentials=None):
    try:
        credentials = credentials or GoogleCredentials.get_application_default()
    except google.auth.exceptions.DefaultCredentialsError:
        return None

    if projects is None:
        resourcemanager = discovery.build('cloudresourcemanager', 'v1', credentials=credentials)
        projects = [project['projectId']
                    for project in resourcemanager.projects().list().execute().get('projects', [])]

    compute = discovery.build('compute', 'v1', credentials=credentials)
    batch = compute.new_batch_http_request()
    results = []
    replies = []
    for project in projects:
        for filter_to_apply in filters:
            batch.add(compute.instances().aggregatedList(project=project,
                                                         filter=filter_to_apply),
                      callback=partial(lambda request_id, result, exception, inside_filter, inside_project: replies.append((inside_project, inside_filter, result)),
                                       inside_filter=filter_to_apply, inside_project=project))
    batch.execute()

    results.extend([result for project, filter, result in replies])
    batch = compute.new_batch_http_request()
    next_batch_requests = [(project, filter, result)
                           for project, filter, result in replies if result and result.get('nextPageToken')]
    replies = []
    while next_batch_requests:
        [batch.add(compute.instances().aggregatedList(project=project,
                                                      filter=filter,
                                                      pageToken=result['nextPageToken']),
                   callback=partial(lambda request_id, result, exception, inside_filter, inside_project: replies.append((inside_project, inside_filter, result)),
                                    inside_filter=filter, inside_project=project))
         for project, filter, result in next_batch_requests]
        batch.execute()
        results.extend([result for project, filter, result in replies])
        batch = compute.new_batch_http_request()
        next_batch_requests = [(project, filter, result)
                               for project, filter, result in replies if result and result.get('nextPageToken')]
        replies = []

    instances = []
    [instances.extend(get_instace_object_from_gcp_list(result['id'].split('/', 2)[1],  # pylint: disable=expression-not-assigned
                                                       raw,
                                                       result.get('items', [])))
     for result in results if result is not None]
    return instances


def gcp_get_instances_by_label(project, name, raw=True, credentials=None):
    return gcp_filter_projects_instances(projects=[project],
                                         filters=['labels.name eq {name}'.format(name=name.replace('*', '.*'))],
                                         raw=raw,
                                         credentials=credentials)


def gcp_get_instances_by_name(project, name, raw=True, credentials=None):
    """Get instances in a GCP region matching name (with * wildcards)."""
    return gcp_filter_projects_instances(projects=[project],
                                         filters=['name eq {name}'.format(name=name.replace('*', '.*'))],
                                         raw=raw,
                                         credentials=credentials)


def get_instace_object_from_gcp_list(project, raw, region_to_instances):
    matching_intsances = []
    for region, region_instances in region_to_instances.iteritems():
        for matching_instance in region_instances.get('instances', []):
            created_by = [item['value'] for item in matching_instance.get(
                'metadata', {}).get('items', []) if item['key'] == 'created-by']
            if created_by and 'instanceGroupManagers' in created_by[0]:
                if raw:
                    autoscaling_group = created_by[0]
                else:
                    autoscaling_group = created_by[0].split('/')[-1]
            else:
                autoscaling_group = None
            public_ip = None
            if raw:
                iam_or_service_account = ','.join([account['email']
                                                   for account in matching_instance['serviceAccounts']])
                instance_type = matching_instance['machineType'].split('/')[-1]
            else:
                iam_or_service_account = ','.join([account['email'].split('@')[0]<commit_msg>Fix empty serviceAccounts in raw format<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""list instances from AWS, and GCP."""
import sys

from argparse import ArgumentParser
import boto3
import botocore
from botocore.exceptions import ClientError
from collections import namedtuple
from concurrent.futures import ThreadPoolExecutor
from dateutil.parser import parse as parse_date
from functools import partial
import kubernetes.client.rest
import kubernetes.config.config_exception
from pytz import UTC
import re
from texttable import Texttable

try:
    from oauth2client.client import GoogleCredentials  # pylint: disable=g-import-not-at-top
    import google.auth.exceptions  # pylint: disable=g-import-not-at-top
    from googleapiclient import discovery  # pylint: disable=g-import-not-at-top
    from googleapiclient import errors  # pylint: disable=g-import-not-at-top
except ImportError:
    print "Warning, google-api-python-client or google-auth not installed, can't connect to GCP instances."

try:
    from kubernetes import client, config  # pylint: disable=g-import-not-at-top,g-multiple-import
except ImportError:
    print "Warning, kubernetes not installed, not getting pods"

AUTOSCALING_GROUP_TAG = u'aws:autoscaling:groupName'
AWS_REGION_LIST = ['us-east-1', 'eu-west-1', 'us-west-2',
                   'sa-east-1', 'ap-southeast-1', 'eu-central-1']
GCP_REGION_LIST = ['us-east1', 'asia-east1', 'asia-northeast1',
                   'asia-southeast1', 'europe-west1', 'us-central1', 'us-west1']
INSTANCE_FILEDS = ['cloud', 'created', 'name', 'region', 'id', 'state', 'ip_address', 'type', 'autoscaling_group',
                   'iam_or_service_account', 'public_dns_name', 'private_ip_address', 'project', 'zone', 'security_groups',
                   'tags', 'vpc_id', 'reservation_type']
DEFAULT_SHOWN_FIELDS = ['project', 'name', 'zone', 'id', 'state', 'ip_address', 'type', 'created',
                        'autoscaling_group', 'iam_or_service_account', 'private_ip_address']
GCP_INSTANCE_TYPE_DICT = {'standard': 'std', 'highmem': 'mem', 'n1-': '', 'highcpu': 'cpu', 'custom': 'ctm'}
ALIGN_OPTIONS = ['l', 'c', 'r']
Instance = namedtuple('Instance', INSTANCE_FILEDS)
SUPPORTED_CLOUDS = ('gcp', 'aws', 'k8s')


def pretify_field(field):
    return ' '.join([word[0].upper() + word[1:] for word in field.split('_')])


def aws_get_instances_by_name(region, name, raw=True):
    """Get all instances in a region matching a name (with wildcards)."""
    return _aws_get_instance_by_tag(region, name, 'tag:Name', raw)


def aws_get_instances_by_autoscaling_group_in_region(region, name, raw=True):
    return _aws_get_instance_by_tag(region, name, 'tag:aws:autoscaling:groupName', raw)


def datetime_to_str(date):
    return date.astimezone(UTC).isoformat().split('+')[0].split('.')[0].replace('T', ' ')


def _aws_instance_from_dict(region, instance, raw):
    tags = {tag['Key']: tag['Value'] for tag in instance.get('Tags', [])}
    return Instance(cloud='aws',  # pylint: disable=expression-not-assigned
                    zone=instance['Placement']['AvailabilityZone'],
                    region=region,
                    id=instance['InstanceId'],
                    name=tags.get('Name', 'Empty'),
                    state=instance['State']['Name'],
                    type=instance['InstanceType'] if raw else instance['InstanceType'].replace('xlarge', 'xl'),
                    autoscaling_group=tags.get(AUTOSCALING_GROUP_TAG, None),
                    public_dns_name=instance['PublicDnsName'],
                    ip_address=instance.get('PublicIpAddress'),
                    iam_or_service_account=instance['IamInstanceProfile'][
                        'Arn'].split('/')[1] if instance.get('IamInstanceProfile') else None,
                    private_ip_address=instance.get('PrivateIpAddress'),
                    project='aws',
                    security_groups=[group['GroupName'] for group in instance['SecurityGroups']],
                    tags=tags,
                    created=datetime_to_str(instance['LaunchTime']),
                    reservation_type=instance.get('InstanceLifecycle', 'on_demand'),
                    vpc_id=instance.get('VpcId'))


def _aws_get_instance_by_tag(region, name, tag, raw):
    """Get all instances matching a tag."""
    client = boto3.session.Session().client('ec2', region)
    matching_reservations = client.describe_instances(Filters=[{'Name': tag, 'Values': [name]}]).get('Reservations', [])
    instances = []
    [[instances.append(_aws_instance_from_dict(region, instance, raw))  # pylint: disable=expression-not-assigned
      for instance in reservation.get('Instances')] for reservation in matching_reservations if reservation]
    return instances


def gcp_filter_projects_instances(projects, filters, raw=True, credentials=None):
    try:
        credentials = credentials or GoogleCredentials.get_application_default()
    except google.auth.exceptions.DefaultCredentialsError:
        return None

    if projects is None:
        resourcemanager = discovery.build('cloudresourcemanager', 'v1', credentials=credentials)
        projects = [project['projectId']
                    for project in resourcemanager.projects().list().execute().get('projects', [])]

    compute = discovery.build('compute', 'v1', credentials=credentials)
    batch = compute.new_batch_http_request()
    results = []
    replies = []
    for project in projects:
        for filter_to_apply in filters:
            batch.add(compute.instances().aggregatedList(project=project,
                                                         filter=filter_to_apply),
                      callback=partial(lambda request_id, result, exception, inside_filter, inside_project: replies.append((inside_project, inside_filter, result)),
                                       inside_filter=filter_to_apply, inside_project=project))
    batch.execute()

    results.extend([result for project, filter, result in replies])
    batch = compute.new_batch_http_request()
    next_batch_requests = [(project, filter, result)
                           for project, filter, result in replies if result and result.get('nextPageToken')]
    replies = []
    while next_batch_requests:
        [batch.add(compute.instances().aggregatedList(project=project,
                                                      filter=filter,
                                                      pageToken=result['nextPageToken']),
                   callback=partial(lambda request_id, result, exception, inside_filter, inside_project: replies.append((inside_project, inside_filter, result)),
                                    inside_filter=filter, inside_project=project))
         for project, filter, result in next_batch_requests]
        batch.execute()
        results.extend([result for project, filter, result in replies])
        batch = compute.new_batch_http_request()
        next_batch_requests = [(project, filter, result)
                               for project, filter, result in replies if result and result.get('nextPageToken')]
        replies = []

    instances = []
    [instances.extend(get_instace_object_from_gcp_list(result['id'].split('/', 2)[1],  # pylint: disable=expression-not-assigned
                                                       raw,
                                                       result.get('items', [])))
     for result in results if result is not None]
    return instances


def gcp_get_instances_by_label(project, name, raw=True, credentials=None):
    return gcp_filter_projects_instances(projects=[project],
                                         filters=['labels.name eq {name}'.format(name=name.replace('*', '.*'))],
                                         raw=raw,
                                         credentials=credentials)


def gcp_get_instances_by_name(project, name, raw=True, credentials=None):
    """Get instances in a GCP region matching name (with * wildcards)."""
    return gcp_filter_projects_instances(projects=[project],
                                         filters=['name eq {name}'.format(name=name.replace('*', '.*'))],
                                         raw=raw,
                                         credentials=credentials)


def get_instace_object_from_gcp_list(project, raw, region_to_instances):
    matching_intsances = []
    for region, region_instances in region_to_instances.iteritems():
        for matching_instance in region_instances.get('instances', []):
            created_by = [item['value'] for item in matching_instance.get(
                'metadata', {}).get('items', []) if item['key'] == 'created-by']
            if created_by and 'instanceGroupManagers' in created_by[0]:
                if raw:
                    autoscaling_group = created_by[0]
                else:
                    autoscaling_group = created_by[0].split('/')[-1]
            else:
                autoscaling_group = None
            public_ip = None
            if raw:
                iam_or_service_account = ','.join([account['email']
                                                   for account in matching_instance.get('serviceAccounts', [])])
                instance_type = matching_instance['machineType'].split('/')[-1]
            else:
                iam_or_service_account = ','.join([account['email'].split('@')[0]
{'fraction_non_alphanumeric': 0.1171906489318823, 'fraction_numerical': 0.004232164449818622, 'mean_word_length': 2.7380414312617702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2108321', 'n_tokens_mistral': 5459, 'n_tokens_neox': 5292, 'n_words': 1252}