<commit_before>/**
 * Copyright (c) 2012-2012 Malhar, Inc. All rights reserved.
 */
package com.datatorrent.stram.cli;

import com.datatorrent.codec.LogicalPlanSerializer;
import com.datatorrent.stram.cli.StramAppLauncher.AppConfig;
import com.datatorrent.stram.cli.StramClientUtils.ClientRMHelper;
import com.datatorrent.stram.cli.StramClientUtils.YarnClientHelper;
import com.datatorrent.stram.plan.logical.*;
import com.datatorrent.stram.security.StramUserLogin;
import com.datatorrent.stram.util.VersionInfo;
import com.datatorrent.stram.util.WebServicesClient;
import com.datatorrent.stram.webapp.StramWebServices;
import com.datatorrent.api.StreamingApplication;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import javax.ws.rs.core.MediaType;

import jline.console.completer.*;
import jline.console.ConsoleReader;
import jline.console.history.History;
import jline.console.history.FileHistory;
import jline.console.history.MemoryHistory;

import org.apache.commons.cli.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.Records;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.tools.ant.DirectoryScanner;
import org.codehaus.jettison.json.JSONException;

/**
 *
 * Provides command line interface for a streaming application on hadoop (yarn)<p>
 *
 * @since 0.3.2
 */
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public class StramCli
{
  private static final Logger LOG = LoggerFactory.getLogger(StramCli.class);
  private final Configuration conf = new Configuration();
  private ClientRMHelper rmClient;
  private ApplicationReport currentApp = null;
  private boolean consolePresent;
  private String[] commandsToExecute;
  private final Map<String, CommandSpec> globalCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, CommandSpec> connectedCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, CommandSpec> logicalPlanChangeCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, String> aliases = new HashMap<String, String>();
  private final Map<String, List<String>> macros = new HashMap<String, List<String>>();
  private boolean changingLogicalPlan = false;
  private List<LogicalPlanRequest> logicalPlanRequestQueue = new ArrayList<LogicalPlanRequest>();
  private FileHistory topLevelHistory;
  private FileHistory changingLogicalPlanHistory;
  private boolean licensedVersion = true;

  public static class Tokenizer
  {
    private static void appendToCommandBuffer(List<String> commandBuffer, StringBuffer buf)
    {
      commandBuffer.add(buf.toString());
      buf.setLength(0);
    }

    private static List<String> startNewCommand(List<List<String>> resultBuffer)
    {
      List<String> newCommand = new ArrayList<String>();
      resultBuffer.add(newCommand);
      return newCommand;
    }

    public static List<String[]> tokenize(String commandLine)
    {
      List<List<String>> resultBuffer = new ArrayList<List<String>>();
      List<String> commandBuffer = startNewCommand(resultBuffer);

      if (commandLine != null) {
        int z = commandLine.length();
        boolean insideQuotes = false;
        boolean potentialEmptyArg = false;
        StringBuffer buf = new StringBuffer();

        for (int i = 0; i < z; ++i) {
          char c = commandLine.charAt(i);
          if (c == '"') {
            potentialEmptyArg = true;
            insideQuotes = !insideQuotes;
          }
          else if (c == '\\') {
            if (z > i + 1) {
              switch (commandLine.charAt(i + 1)) {
                case 'n':
                  buf.append("\n");
                  break;
                case 't':
                  buf.append("\t");
                  break;
                case 'r':
                  buf.append("\r");
                  break;
                case 'b':
                  buf.append("\b");
                  break;
                case 'f':
                  buf.append("\f");
                  break;
                default:
                  buf.append(commandLine.charAt(i + 1));
              }
              ++i;
            }
          }
          else {
            if (insideQuotes) {
              buf.append(c);
            }
            else {
              if (c == ';') {
                commandBuffer = startNewCommand(resultBuffer);
              }
              else if (Character.isWhitespace(c)) {
                if (potentialEmptyArg || buf.length() > 0) {
                  appendToCommandBuffer(commandBuffer, buf);
                }
                potentialEmptyArg = false;
              }
              else {
                buf.append(c);
              }
            }
          }
        }
        if (potentialEmptyArg || buf.length() > 0) {
          appendToCommandBuffer(commandBuffer, buf);
        }
      }

      List<String[]> result = new ArrayList<String[]>();
      for (List<String> command : resultBuffer) {<commit_msg>SPOI-835 #resolve space and escape character support for CLI<commit_after>/**
 * Copyright (c) 2012-2012 Malhar, Inc. All rights reserved.
 */
package com.datatorrent.stram.cli;

import com.datatorrent.codec.LogicalPlanSerializer;
import com.datatorrent.stram.cli.StramAppLauncher.AppConfig;
import com.datatorrent.stram.cli.StramClientUtils.ClientRMHelper;
import com.datatorrent.stram.cli.StramClientUtils.YarnClientHelper;
import com.datatorrent.stram.plan.logical.*;
import com.datatorrent.stram.security.StramUserLogin;
import com.datatorrent.stram.util.VersionInfo;
import com.datatorrent.stram.util.WebServicesClient;
import com.datatorrent.stram.webapp.StramWebServices;
import com.datatorrent.api.StreamingApplication;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import javax.ws.rs.core.MediaType;

import jline.console.completer.*;
import jline.console.ConsoleReader;
import jline.console.history.History;
import jline.console.history.FileHistory;
import jline.console.history.MemoryHistory;

import org.apache.commons.cli.*;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.util.Records;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.tools.ant.DirectoryScanner;
import org.codehaus.jettison.json.JSONException;

/**
 *
 * Provides command line interface for a streaming application on hadoop (yarn)<p>
 *
 * @since 0.3.2
 */
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public class StramCli
{
  private static final Logger LOG = LoggerFactory.getLogger(StramCli.class);
  private final Configuration conf = new Configuration();
  private ClientRMHelper rmClient;
  private ApplicationReport currentApp = null;
  private boolean consolePresent;
  private String[] commandsToExecute;
  private final Map<String, CommandSpec> globalCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, CommandSpec> connectedCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, CommandSpec> logicalPlanChangeCommands = new TreeMap<String, CommandSpec>();
  private final Map<String, String> aliases = new HashMap<String, String>();
  private final Map<String, List<String>> macros = new HashMap<String, List<String>>();
  private boolean changingLogicalPlan = false;
  private List<LogicalPlanRequest> logicalPlanRequestQueue = new ArrayList<LogicalPlanRequest>();
  private FileHistory topLevelHistory;
  private FileHistory changingLogicalPlanHistory;
  private boolean licensedVersion = true;

  public static class Tokenizer
  {
    private static void appendToCommandBuffer(List<String> commandBuffer, StringBuffer buf, boolean potentialEmptyArg)
    {
      if (potentialEmptyArg || buf.length() > 0) {
        commandBuffer.add(buf.toString());
        buf.setLength(0);
      }
    }

    private static List<String> startNewCommand(List<List<String>> resultBuffer)
    {
      List<String> newCommand = new ArrayList<String>();
      resultBuffer.add(newCommand);
      return newCommand;
    }

    public static List<String[]> tokenize(String commandLine)
    {
      List<List<String>> resultBuffer = new ArrayList<List<String>>();
      List<String> commandBuffer = startNewCommand(resultBuffer);

      if (commandLine != null) {
        int z = commandLine.length();
        boolean insideQuotes = false;
        boolean potentialEmptyArg = false;
        StringBuffer buf = new StringBuffer();

        for (int i = 0; i < z; ++i) {
          char c = commandLine.charAt(i);
          if (c == '"') {
            potentialEmptyArg = true;
            insideQuotes = !insideQuotes;
          }
          else if (c == '\\') {
            if (z > i + 1) {
              switch (commandLine.charAt(i + 1)) {
                case 'n':
                  buf.append("\n");
                  break;
                case 't':
                  buf.append("\t");
                  break;
                case 'r':
                  buf.append("\r");
                  break;
                case 'b':
                  buf.append("\b");
                  break;
                case 'f':
                  buf.append("\f");
                  break;
                default:
                  buf.append(commandLine.charAt(i + 1));
              }
              ++i;
            }
          }
          else {
            if (insideQuotes) {
              buf.append(c);
            }
            else {
              if (c == ';') {
                appendToCommandBuffer(commandBuffer, buf, potentialEmptyArg);
                commandBuffer = startNewCommand(resultBuffer);
              }
              else if (Character.isWhitespace(c)) {
                appendToCommandBuffer(commandBuffer, buf, potentialEmptyArg);
                potentialEmptyArg = false;
              }
              else {
                buf.append(c);
              }
            }
          }
        }
        appendToCommandBuffer(commandBuffer, buf, potentialEmptyArg);
      }

      List<String[]> result = new ArrayList<String[]>();
      for (List<String> command : resultBuffer) {
{'fraction_non_alphanumeric': 0.10768026441680438, 'fraction_numerical': 0.003653126902670262, 'mean_word_length': 2.757516339869281, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2402132', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3168, 'n_words': 669}