<commit_before>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package se.kth.karamel.backend.launcher.amazon;

import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
import org.jclouds.aws.ec2.features.AWSSecurityGroupApi;
import org.jclouds.aws.ec2.options.CreateSecurityGroupOptions;
import org.jclouds.compute.RunNodesException;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.TemplateBuilder;
import org.jclouds.ec2.domain.KeyPair;
import org.jclouds.ec2.domain.SecurityGroup;
import org.jclouds.ec2.features.SecurityGroupApi;
import org.jclouds.net.domain.IpPermission;
import org.jclouds.net.domain.IpProtocol;
import org.jclouds.rest.AuthorizationException;
import se.kth.karamel.backend.converter.UserClusterDataExtractor;
import se.kth.karamel.backend.launcher.Launcher;
import se.kth.karamel.backend.running.model.ClusterRuntime;
import se.kth.karamel.backend.running.model.GroupRuntime;
import se.kth.karamel.backend.running.model.MachineRuntime;
import se.kth.karamel.common.Settings;
import se.kth.karamel.common.exception.KaramelException;
import se.kth.karamel.client.model.Ec2;
import se.kth.karamel.client.model.Provider;
import se.kth.karamel.client.model.json.JsonCluster;
import se.kth.karamel.client.model.json.JsonGroup;
import se.kth.karamel.common.Confs;
import se.kth.karamel.common.Ec2Credentials;
import se.kth.karamel.common.SshKeyPair;
import se.kth.karamel.common.exception.InvalidEc2CredentialsException;

/**
 * @author kamal
 */
public final class Ec2Launcher extends Launcher {

  private static final Logger logger = Logger.getLogger(Ec2Launcher.class);
  public static boolean TESTING = true;
  public final Ec2Context context;
  public final SshKeyPair sshKeyPair;

  public Ec2Launcher(Ec2Context context, SshKeyPair sshKeyPair) {
    this.context = context;
    this.sshKeyPair = sshKeyPair;
    logger.info(String.format("Access-key='%s'", context.getCredentials().getAccessKey()));
    logger.info(String.format("Public-key='%s'", sshKeyPair.getPublicKeyPath()));
    logger.info(String.format("Private-key='%s'", sshKeyPair.getPrivateKeyPath()));
  }

  public static Ec2Context validateCredentials(Ec2Credentials credentials) throws InvalidEc2CredentialsException {
    try {
      if (credentials.getAccessKey().isEmpty() || credentials.getSecretKey().isEmpty()) {
        throw new InvalidEc2CredentialsException("Ec2 credentials empty - not entered yet.");
      }
      Ec2Context cxt = new Ec2Context(credentials);
      SecurityGroupApi securityGroupApi = cxt.getSecurityGroupApi();
      securityGroupApi.describeSecurityGroupsInRegion(Settings.PROVIDER_EC2_DEFAULT_REGION);
      return cxt;
    } catch (AuthorizationException e) {
      throw new InvalidEc2CredentialsException(e.getMessage() + " - accountid:" + credentials.getAccessKey(), e);
    }
  }

  public static Ec2Credentials readCredentials(Confs confs) {  
    String accessKey =  System.getenv(Settings.AWS_ACCESS_KEY_ENV_VAR);
    String secretKey = System.getenv(Settings.AWS_SECRET_KEY_ENV_VAR);

    if (accessKey == null || accessKey.isEmpty()) {
      accessKey = confs.getProperty(Settings.AWS_ACCESS_KEY);
    }
    if (secretKey == null || secretKey.isEmpty()) {
      secretKey = confs.getProperty(Settings.AWS_SECRET_KEY);
    }
    Ec2Credentials credentials = null;
    if (accessKey != null && !accessKey.isEmpty() && secretKey != null && !accessKey.isEmpty()) {
      credentials = new Ec2Credentials();
      credentials.setAccessKey(accessKey);
      credentials.setSecretKey(secretKey);
    }
    return credentials;
  }

  @Override
  public String forkGroup(JsonCluster definition, ClusterRuntime runtime, String groupName) throws KaramelException {
    JsonGroup jg = UserClusterDataExtractor.findGroup(definition, groupName);
    Provider provider = UserClusterDataExtractor.getGroupProvider(definition, groupName);
    Ec2 ec2 = (Ec2) provider;
    Set<String> ports = new HashSet<>();
    ports.addAll(Settings.EC2_DEFAULT_PORTS);
    String groupId = createSecurityGroup(definition.getName(), jg.getName(), ec2, ports);
    return groupId;
  }

  public String createSecurityGroup(String clusterName, String groupName, Ec2 ec2, Set<String> ports) 
      throws KaramelException {
    String uniqeGroupName = Settings.EC2_UNIQUE_GROUP_NAME(clusterName, groupName);
    logger.info(String.format("Creating security group '%s' ...", uniqeGroupName));
    if (context == null) {
      throw new KaramelException("Register your valid credentials first :-| ");
    }

    if (sshKeyPair == null) {
      throw new KaramelException("Choose your ssh keypair first :-| ");
    }

    Optional<? extends org.jclouds.ec2.features.SecurityGroupApi> securityGroupExt
        = context.getEc2api().getSecurityGroupApiForRegion(ec2.getRegion());
    if (securityGroupExt.isPresent()) {
      AWSSecurityGroupApi client = (AWSSecurityGroupApi) securityGroupExt.get();
      String groupId = null;
      if (ec2.getVpc() != null) {
        CreateSecurityGroupOptions csgos = CreateSecurityGroupOptions.Builder.vpcId(ec2.getVpc());
        groupId = client.createSecurityGroupInRegionAndReturnId(ec2.getRegion(), uniqeGroupName, uniqeGroupName, csgos);
      } else {
        groupId = client.createSecurityGroupInRegionAndReturnId(ec2.getRegion(), uniqeGroupName, uniqeGroupName);
      }

      if (!TESTING) {
        for (String port : ports) {
          Integer p = null;
          IpProtocol pr = null;
          if (port.contains("/")) {
            String[] s = port.split("/");
            p = Integer.valueOf(s[0]);
            pr = IpProtocol.valueOf(s[1]);
          } else {
            p = Integer.valueOf(port);
            pr = IpProtocol.TCP;
          }
          client.authorizeSecurityGroupIngressInRegion(ec2.getRegion(),
              uniqeGroupName, pr, p, Integer.valueOf(port), "0.0.0.0/0");
          logger.info(String.format("Ports became open for '%s'", uniqeGroupName));
        }
      } else {
        IpPermission ippermission = IpPermission.builder().ipProtocol(IpProtocol.TCP).fromPort(0).toPort(65535).
            cidrBlock("0.0.0.0/0").build();
        client.authorizeSecurityGroupIngressInRegion(ec2.getRegion(), groupId, ippermission);
        logger.info(String.format("Ports became open for '%s'", uniqeGroupName));
      }
      logger.info(String.format("Security group '%s' was created :)", uniqeGroupName));
      return groupId;
    }<commit_msg>opening upd ports on aws vms<commit_after>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package se.kth.karamel.backend.launcher.amazon;

import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;
import org.jclouds.aws.AWSResponseException;
import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
import org.jclouds.aws.ec2.features.AWSSecurityGroupApi;
import org.jclouds.aws.ec2.options.CreateSecurityGroupOptions;
import org.jclouds.compute.RunNodesException;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.TemplateBuilder;
import org.jclouds.ec2.domain.KeyPair;
import org.jclouds.ec2.domain.SecurityGroup;
import org.jclouds.ec2.features.SecurityGroupApi;
import org.jclouds.net.domain.IpPermission;
import org.jclouds.net.domain.IpProtocol;
import org.jclouds.rest.AuthorizationException;
import se.kth.karamel.backend.converter.UserClusterDataExtractor;
import se.kth.karamel.backend.launcher.Launcher;
import se.kth.karamel.backend.running.model.ClusterRuntime;
import se.kth.karamel.backend.running.model.GroupRuntime;
import se.kth.karamel.backend.running.model.MachineRuntime;
import se.kth.karamel.common.Settings;
import se.kth.karamel.common.exception.KaramelException;
import se.kth.karamel.client.model.Ec2;
import se.kth.karamel.client.model.Provider;
import se.kth.karamel.client.model.json.JsonCluster;
import se.kth.karamel.client.model.json.JsonGroup;
import se.kth.karamel.common.Confs;
import se.kth.karamel.common.Ec2Credentials;
import se.kth.karamel.common.SshKeyPair;
import se.kth.karamel.common.exception.InvalidEc2CredentialsException;

/**
 * @author kamal
 */
public final class Ec2Launcher extends Launcher {

  private static final Logger logger = Logger.getLogger(Ec2Launcher.class);
  public static boolean TESTING = true;
  public final Ec2Context context;
  public final SshKeyPair sshKeyPair;

  public Ec2Launcher(Ec2Context context, SshKeyPair sshKeyPair) {
    this.context = context;
    this.sshKeyPair = sshKeyPair;
    logger.info(String.format("Access-key='%s'", context.getCredentials().getAccessKey()));
    logger.info(String.format("Public-key='%s'", sshKeyPair.getPublicKeyPath()));
    logger.info(String.format("Private-key='%s'", sshKeyPair.getPrivateKeyPath()));
  }

  public static Ec2Context validateCredentials(Ec2Credentials credentials) throws InvalidEc2CredentialsException {
    try {
      if (credentials.getAccessKey().isEmpty() || credentials.getSecretKey().isEmpty()) {
        throw new InvalidEc2CredentialsException("Ec2 credentials empty - not entered yet.");
      }
      Ec2Context cxt = new Ec2Context(credentials);
      SecurityGroupApi securityGroupApi = cxt.getSecurityGroupApi();
      securityGroupApi.describeSecurityGroupsInRegion(Settings.PROVIDER_EC2_DEFAULT_REGION);
      return cxt;
    } catch (AuthorizationException e) {
      throw new InvalidEc2CredentialsException(e.getMessage() + " - accountid:" + credentials.getAccessKey(), e);
    }
  }

  public static Ec2Credentials readCredentials(Confs confs) {  
    String accessKey =  System.getenv(Settings.AWS_ACCESS_KEY_ENV_VAR);
    String secretKey = System.getenv(Settings.AWS_SECRET_KEY_ENV_VAR);

    if (accessKey == null || accessKey.isEmpty()) {
      accessKey = confs.getProperty(Settings.AWS_ACCESS_KEY);
    }
    if (secretKey == null || secretKey.isEmpty()) {
      secretKey = confs.getProperty(Settings.AWS_SECRET_KEY);
    }
    Ec2Credentials credentials = null;
    if (accessKey != null && !accessKey.isEmpty() && secretKey != null && !accessKey.isEmpty()) {
      credentials = new Ec2Credentials();
      credentials.setAccessKey(accessKey);
      credentials.setSecretKey(secretKey);
    }
    return credentials;
  }

  @Override
  public String forkGroup(JsonCluster definition, ClusterRuntime runtime, String groupName) throws KaramelException {
    JsonGroup jg = UserClusterDataExtractor.findGroup(definition, groupName);
    Provider provider = UserClusterDataExtractor.getGroupProvider(definition, groupName);
    Ec2 ec2 = (Ec2) provider;
    Set<String> ports = new HashSet<>();
    ports.addAll(Settings.EC2_DEFAULT_PORTS);
    String groupId = createSecurityGroup(definition.getName(), jg.getName(), ec2, ports);
    return groupId;
  }

  public String createSecurityGroup(String clusterName, String groupName, Ec2 ec2, Set<String> ports) 
      throws KaramelException {
    String uniqeGroupName = Settings.EC2_UNIQUE_GROUP_NAME(clusterName, groupName);
    logger.info(String.format("Creating security group '%s' ...", uniqeGroupName));
    if (context == null) {
      throw new KaramelException("Register your valid credentials first :-| ");
    }

    if (sshKeyPair == null) {
      throw new KaramelException("Choose your ssh keypair first :-| ");
    }

    Optional<? extends org.jclouds.ec2.features.SecurityGroupApi> securityGroupExt
        = context.getEc2api().getSecurityGroupApiForRegion(ec2.getRegion());
    if (securityGroupExt.isPresent()) {
      AWSSecurityGroupApi client = (AWSSecurityGroupApi) securityGroupExt.get();
      String groupId = null;
      if (ec2.getVpc() != null) {
        CreateSecurityGroupOptions csgos = CreateSecurityGroupOptions.Builder.vpcId(ec2.getVpc());
        groupId = client.createSecurityGroupInRegionAndReturnId(ec2.getRegion(), uniqeGroupName, uniqeGroupName, csgos);
      } else {
        groupId = client.createSecurityGroupInRegionAndReturnId(ec2.getRegion(), uniqeGroupName, uniqeGroupName);
      }

      if (!TESTING) {
        for (String port : ports) {
          Integer p = null;
          IpProtocol pr = null;
          if (port.contains("/")) {
            String[] s = port.split("/");
            p = Integer.valueOf(s[0]);
            pr = IpProtocol.valueOf(s[1]);
          } else {
            p = Integer.valueOf(port);
            pr = IpProtocol.TCP;
          }
          client.authorizeSecurityGroupIngressInRegion(ec2.getRegion(),
              uniqeGroupName, pr, p, Integer.valueOf(port), "0.0.0.0/0");
          logger.info(String.format("Ports became open for '%s'", uniqeGroupName));
        }
      } else {
        IpPermission tcpPerms = IpPermission.builder().ipProtocol(IpProtocol.TCP).
            fromPort(0).toPort(65535).cidrBlock("0.0.0.0/0").build();
        IpPermission udpPerms = IpPermission.builder().ipProtocol(IpProtocol.UDP).
            fromPort(0).toPort(65535).cidrBlock("0.0.0.0/0").build();
        ArrayList<IpPermission> perms = Lists.newArrayList(tcpPerms, udpPerms);
        client.authorizeSecurityGroupIngressInRegion(ec2.getRegion(), groupId, perms);
        logger.info(String.format("Ports became open for '%s'", uniqeGroupName));
      }
      logger.info(String.format("Security group '%s' was created :)", uniqeGroupName));
      return groupId;
    }
{'fraction_non_alphanumeric': 0.11622481442205726, 'fraction_numerical': 0.009685401201838105, 'mean_word_length': 4.155247813411079, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5329050', 'n_tokens_mistral': 4388, 'n_tokens_neox': 4300, 'n_words': 783}