<commit_before>#!/usr/bin/env python

from __future__ import print_function

import argparse
import base64
import json
import os
import subprocess
import sys
import time

from aws_utils.instance_utils import format_instance, AwsInstance

# Ithemal runs on Python 2 mostly
try:
    input = raw_input
except NameError:
    pass

_DIRNAME = os.path.abspath(os.path.dirname(__file__))

class InstanceMaker(AwsInstance):
    def __init__(self, identity, name, instance_type, db, force, no_connect, spot, queue):
        super(InstanceMaker, self).__init__(identity, require_pem=True)
        self.name = name
        self.instance_type = instance_type
        self.db = db
        self.force = force
        self.no_connect = no_connect
        self.spot = spot
        self.queue_url = queue

    def start_instance(self):
        if not self.force:
            running_instances = self.get_running_instances()
            if running_instances:
                print('You already have {} running instances:'.format(len(running_instances)))
                for instance in running_instances:
                    print(format_instance(instance))
                try:
                    res = input('Would you still like to continue? (y/n) ').lower()[0]
                except KeyboardInterrupt:
                    print('Not creating a new instance')
                    return

                if res[0] != 'y':
                    print('Not creating a new instance')
                    return

        name = 'Ithemal'
        if self.name:
            name += ': {}'.format(self.name)

        block_device_mappings = [{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 16}}]
        iam_profile_name = 'ithemal-ec2'

        if self.spot:
            launch_specification = {
                'InstanceType': self.instance_type,
                'SecurityGroupIds': ['<KEY>'],
                'BlockDeviceMappings': block_device_mappings,
                'KeyName': self.identity,
                'ImageId': 'ami-0b59bfac6be064b78',
                'IamInstanceProfile': {'Name': iam_profile_name},
            }
            run_com = lambda com: json.loads(subprocess.check_output(com))['SpotInstanceRequests'][0]
            com = [
                'aws', 'ec2', 'request-spot-instances',
                '--launch-specification', json.dumps(launch_specification)
            ]
            if self.spot > 0:
                com.extend(['--block-duration-minutes', str(self.spot * 60)])
            output = run_com(com)
            print('Submitted spot instance request')

            try:
                while 'InstanceId' not in output:
                    print('\rWaiting for spot request to be fulfilled ({})...'.format(
                        output['Status']['Code']
                    ), end=' ' * 20 + '\r')

                    time.sleep(1)
                    output = run_com([
                        'aws', 'ec2', 'describe-spot-instance-requests',
                        '--spot-instance-request-ids', output['SpotInstanceRequestId'],
                    ])

            except KeyboardInterrupt:
                subprocess.check_call([
                    'aws', 'ec2', 'cancel-spot-instance-requests',
                    '--spot-instance-request-ids', output['SpotInstanceRequestId'],
                ])
                sys.exit(1)

            print() # clear status message

            instance_id = output['InstanceId']
            # set the name, since spot instances don't let us do that in the creation request
            subprocess.check_call([
                'aws', 'ec2', 'create-tags',
                '--resources', instance_id,
                '--tags', 'Key=Name,Value="{}"'.format(name)
            ])
        else:
            args = [
                'aws', 'ec2', 'run-instances',
                '--instance-type', self.instance_type,
                '--key-name', self.identity,
                '--image-id', 'ami-0b59bfac6be064b78',
                '--tag-specifications', 'ResourceType="instance",Tags=[{{Key="Name",Value="{}"}}]'.format(name),
                '--security-group-ids', 'sg-0780fe1760c00d96d',
                '--block-device-mappings', json.dumps(block_device_mappings),
                '--iam-instance-profile', iam_profile_name,
            ]
            output = subprocess.check_output(args)
            parsed_output = json.loads(output)
            instance = parsed_output['Instances'][0]
            instance_id = instance['InstanceId']

        print('Started instance! Waiting for connection...')

        subprocess.check_call(['aws', 'ec2', 'wait', 'instance-running', '--instance-ids', instance_id])

        instance = next(instance for instance in self.get_running_instances() if instance['InstanceId'] == instance_id)

        # wait for SSH to actually become available
        while subprocess.call(['ssh', '-oStrictHostKeyChecking=no', '-i', self.pem_key, 'ec2-user@{}'.format(instance['PublicDnsName']), 'exit'],
                              stdout=open(os.devnull, 'w'),
                              stderr=open(os.devnull, 'w'),
        ):
            time.sleep(1)


        git_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=_DIRNAME).strip()
        ls_files = subprocess.Popen(['git', 'ls-files'], cwd=git_root, stdout=subprocess.PIPE)
        tar = subprocess.Popen(['tar', 'Tcz', '-'], cwd=git_root, stdin=ls_files.stdout, stdout=subprocess.PIPE)

        aws_credentials = json.loads(subprocess.check_output(['aws', 'ecr', 'get-authorization-token']).strip())
        authorization_datum = aws_credentials['authorizationData'][0]
        aws_authorization = base64.b64decode(authorization_datum['authorizationToken'])
        aws_authorization_user = aws_authorization[:aws_authorization.index(':')]<commit_msg>start instance with iam profile correctly<commit_after>#!/usr/bin/env python

from __future__ import print_function

import argparse
import base64
import json
import os
import subprocess
import sys
import time

from aws_utils.instance_utils import format_instance, AwsInstance

# Ithemal runs on Python 2 mostly
try:
    input = raw_input
except NameError:
    pass

_DIRNAME = os.path.abspath(os.path.dirname(__file__))

class InstanceMaker(AwsInstance):
    def __init__(self, identity, name, instance_type, db, force, no_connect, spot, queue):
        super(InstanceMaker, self).__init__(identity, require_pem=True)
        self.name = name
        self.instance_type = instance_type
        self.db = db
        self.force = force
        self.no_connect = no_connect
        self.spot = spot
        self.queue_url = queue

    def start_instance(self):
        if not self.force:
            running_instances = self.get_running_instances()
            if running_instances:
                print('You already have {} running instances:'.format(len(running_instances)))
                for instance in running_instances:
                    print(format_instance(instance))
                try:
                    res = input('Would you still like to continue? (y/n) ').lower()[0]
                except KeyboardInterrupt:
                    print('Not creating a new instance')
                    return

                if res[0] != 'y':
                    print('Not creating a new instance')
                    return

        name = 'Ithemal'
        if self.name:
            name += ': {}'.format(self.name)

        block_device_mappings = [{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 16}}]
        iam_profile_name = 'ithemal-ec2'
        iam_profile_struct = {'Name': iam_profile_name}

        if self.spot:
            launch_specification = {
                'InstanceType': self.instance_type,
                'SecurityGroupIds': ['sg-0780fe1760c00d96d'],
                'BlockDeviceMappings': block_device_mappings,
                'KeyName': self.identity,
                'ImageId': 'ami-0b59bfac6be064b78',
                'IamInstanceProfile': iam_profile_struct,
            }
            run_com = lambda com: json.loads(subprocess.check_output(com))['SpotInstanceRequests'][0]
            com = [
                'aws', 'ec2', 'request-spot-instances',
                '--launch-specification', json.dumps(launch_specification)
            ]
            if self.spot > 0:
                com.extend(['--block-duration-minutes', str(self.spot * 60)])
            output = run_com(com)
            print('Submitted spot instance request')

            try:
                while 'InstanceId' not in output:
                    print('\rWaiting for spot request to be fulfilled ({})...'.format(
                        output['Status']['Code']
                    ), end=' ' * 20 + '\r')

                    time.sleep(1)
                    output = run_com([
                        'aws', 'ec2', 'describe-spot-instance-requests',
                        '--spot-instance-request-ids', output['SpotInstanceRequestId'],
                    ])

            except KeyboardInterrupt:
                subprocess.check_call([
                    'aws', 'ec2', 'cancel-spot-instance-requests',
                    '--spot-instance-request-ids', output['SpotInstanceRequestId'],
                ])
                sys.exit(1)

            print() # clear status message

            instance_id = output['InstanceId']
            # set the name, since spot instances don't let us do that in the creation request
            subprocess.check_call([
                'aws', 'ec2', 'create-tags',
                '--resources', instance_id,
                '--tags', 'Key=Name,Value="{}"'.format(name)
            ])
        else:
            args = [
                'aws', 'ec2', 'run-instances',
                '--instance-type', self.instance_type,
                '--key-name', self.identity,
                '--image-id', 'ami-0b59bfac6be064b78',
                '--tag-specifications', 'ResourceType="instance",Tags=[{{Key="Name",Value="{}"}}]'.format(name),
                '--security-group-ids', 'sg-0780fe1760c00d96d',
                '--block-device-mappings', json.dumps(block_device_mappings),
                '--iam-instance-profile', json.dumps(iam_profile_struct),
            ]
            output = subprocess.check_output(args)
            parsed_output = json.loads(output)
            instance = parsed_output['Instances'][0]
            instance_id = instance['InstanceId']

        print('Started instance! Waiting for connection...')

        subprocess.check_call(['aws', 'ec2', 'wait', 'instance-running', '--instance-ids', instance_id])

        instance = next(instance for instance in self.get_running_instances() if instance['InstanceId'] == instance_id)

        # wait for SSH to actually become available
        while subprocess.call(['ssh', '-oStrictHostKeyChecking=no', '-i', self.pem_key, 'ec2-user@{}'.format(instance['PublicDnsName']), 'exit'],
                              stdout=open(os.devnull, 'w'),
                              stderr=open(os.devnull, 'w'),
        ):
            time.sleep(1)


        git_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], cwd=_DIRNAME).strip()
        ls_files = subprocess.Popen(['git', 'ls-files'], cwd=git_root, stdout=subprocess.PIPE)
        tar = subprocess.Popen(['tar', 'Tcz', '-'], cwd=git_root, stdin=ls_files.stdout, stdout=subprocess.PIPE)

        aws_credentials = json.loads(subprocess.check_output(['aws', 'ecr', 'get-authorization-token']).strip())
        authorization_datum = aws_credentials['authorizationData'][0]
        aws_authorization = base64.b64decode(authorization_datum['authorizationToken'])
        aws_authorization_user = aws_authorization[:aws_authorization.index(':')]
{'fraction_non_alphanumeric': 0.12779145792646684, 'fraction_numerical': 0.011208287339729983, 'mean_word_length': 2.0084291187739463, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3670508', 'n_tokens_mistral': 3421, 'n_tokens_neox': 3233, 'n_words': 650}