<commit_before>package com.ldaniels528.verify.modules.core

import java.io.{File, PrintStream}
import java.util.{Date, TimeZone}

import com.ldaniels528.verify.modules.CommandParser.UnixLikeArgs
import com.ldaniels528.verify.modules.ModuleManager.ModuleVariable
import com.ldaniels528.verify.modules.{Command, Module, SimpleParams, UnixLikeParams}
import com.ldaniels528.verify.support.avro.AvroReading
import com.ldaniels528.verify.util.VxUtils._
import com.ldaniels528.verify.vscript.VScriptRuntime.ConstantValue
import com.ldaniels528.verify.vscript.{OpCode, Scope, Variable}
import com.ldaniels528.verify.{SessionManagement, VerifyShell, VxRuntimeContext}
import org.apache.commons.io.IOUtils

import scala.concurrent.ExecutionContext.Implicits._
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.language.postfixOps
import scala.util.Properties

/**
 * Core Module
 * @author <NAME> <<EMAIL>>
 */
class CoreModule(rt: VxRuntimeContext) extends Module with AvroReading {
  private implicit val rtc: VxRuntimeContext = rt
  private implicit val scope: Scope = rt.scope
  private implicit val out: PrintStream = rt.out

  // define the process parsing regular expression
  private val PID_MacOS_r = "^\\s*(\\d+)\\s*(\\d+)\\s*(\\d+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r
  private val PID_Linux_r = "^\\s*(\\S+)\\s*(\\d+)\\s*(\\d+)\\s*(\\d+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r
  private val NET_STAT_r = "^\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r

  override def moduleName = "core"

  override def getCommands: Seq[Command] = Seq(
    Command(this, "!", executeHistory, SimpleParams(Seq("index"), Seq.empty), help = "Executes a previously issued command"),
    Command(this, "autoswitch", autoSwitch, SimpleParams(Seq.empty, Seq("state")), help = "Automatically switches to the module of the most recently executed command"),
    Command(this, "?", help, SimpleParams(Nil, Seq("search-term")), help = "Provides the list of available commands"),
    Command(this, "avcat", avroCat, SimpleParams(required = Seq("variable")), help = "Displays the contents of a schema variable", promptAware = false),
    Command(this, "avload", avroLoadSchema, SimpleParams(required = Seq("variable", "schemaPath")), help = "Loads an Avro schema into memory", promptAware = false),
    Command(this, "cat", cat, SimpleParams(Seq("file"), Seq.empty), help = "Dumps the contents of the given file", promptAware = true),
    Command(this, "cd", changeDir, SimpleParams(Seq("path"), Seq.empty), help = "Changes the local file system path/directory", promptAware = true),
    Command(this, "charset", charSet, SimpleParams(Seq.empty, Seq("encoding")), help = "Retrieves or sets the character encoding"),
    Command(this, "class", inspectClass, SimpleParams(Seq.empty, Seq("action")), help = "Inspects a class using reflection"),
    Command(this, "columns", columnWidthGetOrSet, SimpleParams(Seq.empty, Seq("columnWidth")), help = "Retrieves or sets the column width for message output"),
    Command(this, "debug", debug, SimpleParams(Seq.empty, Seq("enabled")), help = "Switches debugging on/off", undocumented = true),
    Command(this, "exit", exit, SimpleParams(), help = "Exits the shell"),
    Command(this, "help", help, SimpleParams(), help = "Provides the list of available commands"),
    Command(this, "history", listHistory, SimpleParams(), help = "Returns a list of previously issued commands"),
    Command(this, "hostname", hostname, SimpleParams(), help = "Returns the name of the host system"),
    Command(this, "jobs", listJobs, SimpleParams(), help = "Returns the list of currently running jobs"),
    Command(this, "ls", listFiles, SimpleParams(Seq.empty, Seq("path")), help = "Retrieves the files from the current directory", promptAware = true),
    Command(this, "modules", listModules, SimpleParams(), help = "Returns a list of configured modules"),
    Command(this, "pkill", processKill, SimpleParams(Seq("pid0"), Seq("pid1", "pid2", "pid3", "pid4", "pid5", "pid6")), help = "Terminates specific running processes"),
    Command(this, "ps", processList, SimpleParams(Seq.empty, Seq("node", "timeout")), help = "Display a list of \"configured\" running processes (EXPERIMENTAL)"),
    Command(this, "pwd", printWorkingDirectory, SimpleParams(Seq.empty, Seq.empty), help = "Display current working directory"),
    Command(this, "resource", findResource, SimpleParams(Seq("resource-name"), Seq.empty), help = "Inspects the classpath for the given resource"),
    Command(this, "runjava", executeJavaApp, SimpleParams(Seq("jarFile", "className"), (1 to 10).map(n => s"arg$n")), help = "Executes a Java class' main method"),
    Command(this, "scope", listScope, SimpleParams(), help = "Returns the contents of the current scope"),
    Command(this, "syntax", syntax, SimpleParams(Seq("command"), Seq.empty), help = "Returns the syntax/usage for a given command"),
    Command(this, "systime", systemTime, SimpleParams(), help = "Returns the system time as an EPOC in milliseconds"),
    Command(this, "time", time, SimpleParams(), help = "Returns the system time"),
    Command(this, "timeutc", timeUTC, SimpleParams(), help = "Returns the system time in UTC"),
    Command(this, "undoc", listUndocumented, UnixLikeParams(), help = "Displays undocumented commands", undocumented = true),
    Command(this, "use", useModule, SimpleParams(Seq("module"), Seq.empty), help = "Switches the active module"),
    Command(this, "version", version, SimpleParams(), help = "Returns the Verify application version"),
    Command(this, "wget", httpGet, SimpleParams(required = Seq("url")), help = "Retrieves remote content via HTTP"))
<commit_msg>clean up and updates<commit_after>package com.ldaniels528.verify.modules.core

import java.io.{File, PrintStream}
import java.util.{Date, TimeZone}

import com.ldaniels528.verify.modules.CommandParser.UnixLikeArgs
import com.ldaniels528.verify.modules.ModuleManager.ModuleVariable
import com.ldaniels528.verify.modules.{Command, Module, SimpleParams, UnixLikeParams}
import com.ldaniels528.verify.support.avro.AvroReading
import com.ldaniels528.verify.util.VxUtils._
import com.ldaniels528.verify.vscript.VScriptRuntime.ConstantValue
import com.ldaniels528.verify.vscript.{OpCode, Scope, Variable}
import com.ldaniels528.verify.{SessionManagement, VerifyShell, VxRuntimeContext}
import org.apache.commons.io.IOUtils

import scala.concurrent.ExecutionContext.Implicits._
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.language.postfixOps
import scala.util.Properties

/**
 * Core Module
 * @author <NAME> <<EMAIL>>
 */
class CoreModule(rt: VxRuntimeContext) extends Module with AvroReading {
  private implicit val rtc: VxRuntimeContext = rt
  private implicit val scope: Scope = rt.scope
  private implicit val out: PrintStream = rt.out

  // define the process parsing regular expression
  private val PID_MacOS_r = "^\\s*(\\d+)\\s*(\\d+)\\s*(\\d+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r
  private val PID_Linux_r = "^\\s*(\\S+)\\s*(\\d+)\\s*(\\d+)\\s*(\\d+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r
  private val NET_STAT_r = "^\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(\\S+)\\s*(.*)".r

  override def moduleName = "core"

  override def getCommands: Seq[Command] = Seq(
    Command(this, "!", executeHistory, UnixLikeParams(Seq("index" -> true, "count" -> false), Nil), help = "Executes a previously issued command"),
    Command(this, "?", help, SimpleParams(Nil, Seq("search-term")), help = "Provides the list of available commands"),
    Command(this, "autoswitch", autoSwitch, SimpleParams(Nil, Seq("state")), help = "Automatically switches to the module of the most recently executed command"),
    Command(this, "avcat", avroCat, SimpleParams(required = Seq("variable")), help = "Displays the contents of a schema variable", promptAware = false),
    Command(this, "avload", avroLoadSchema, SimpleParams(required = Seq("variable", "schemaPath")), help = "Loads an Avro schema into memory", promptAware = false),
    Command(this, "cat", cat, SimpleParams(Seq("file"), Nil), help = "Dumps the contents of the given file", promptAware = true),
    Command(this, "cd", changeDir, SimpleParams(Seq("path"), Nil), help = "Changes the local file system path/directory", promptAware = true),
    Command(this, "charset", charSet, SimpleParams(Nil, Seq("encoding")), help = "Retrieves or sets the character encoding"),
    Command(this, "class", inspectClass, SimpleParams(Nil, Seq("action")), help = "Inspects a class using reflection"),
    Command(this, "columns", columnWidthGetOrSet, SimpleParams(Nil, Seq("columnWidth")), help = "Retrieves or sets the column width for message output"),
    Command(this, "debug", debug, SimpleParams(Nil, Seq("enabled")), help = "Switches debugging on/off", undocumented = true),
    Command(this, "exit", exit, SimpleParams(), help = "Exits the shell"),
    Command(this, "help", help, SimpleParams(), help = "Provides the list of available commands"),
    Command(this, "history", listHistory, SimpleParams(Nil, Seq("count")), help = "Returns a list of previously issued commands"),
    Command(this, "hostname", hostname, SimpleParams(), help = "Returns the name of the host system"),
    Command(this, "jobs", listJobs, SimpleParams(), help = "Returns the list of currently running jobs"),
    Command(this, "ls", listFiles, SimpleParams(Nil, Seq("path")), help = "Retrieves the files from the current directory", promptAware = true),
    Command(this, "modules", listModules, SimpleParams(), help = "Returns a list of configured modules"),
    Command(this, "pkill", processKill, SimpleParams(Seq("pid0"), Seq("pid1", "pid2", "pid3", "pid4", "pid5", "pid6")), help = "Terminates specific running processes"),
    Command(this, "ps", processList, SimpleParams(Nil, Seq("node", "timeout")), help = "Display a list of \"configured\" running processes (EXPERIMENTAL)"),
    Command(this, "pwd", printWorkingDirectory, SimpleParams(Nil, Nil), help = "Display current working directory"),
    Command(this, "resource", findResource, SimpleParams(Seq("resource-name"), Nil), help = "Inspects the classpath for the given resource"),
    Command(this, "runjava", executeJavaApp, SimpleParams(Seq("jarFile", "className"), (1 to 10).map(n => s"arg$n")), help = "Executes a Java class' main method"),
    Command(this, "scope", listScope, SimpleParams(), help = "Returns the contents of the current scope"),
    Command(this, "syntax", syntax, SimpleParams(Seq("command"), Nil), help = "Returns the syntax/usage for a given command"),
    Command(this, "systime", systemTime, SimpleParams(), help = "Returns the system time as an EPOC in milliseconds"),
    Command(this, "time", time, SimpleParams(), help = "Returns the system time"),
    Command(this, "timeutc", timeUTC, SimpleParams(), help = "Returns the system time in UTC"),
    Command(this, "undoc", listUndocumented, UnixLikeParams(), help = "Displays undocumented commands", undocumented = true),
    Command(this, "use", useModule, SimpleParams(Seq("module"), Nil), help = "Switches the active module"),
    Command(this, "version", version, SimpleParams(), help = "Returns the Verify application version"),
    Command(this, "wget", httpGet, SimpleParams(required = Seq("url")), help = "Retrieves remote content via HTTP"))

{'fraction_non_alphanumeric': 0.1660545645330535, 'fraction_numerical': 0.006470793983910458, 'mean_word_length': 5.378694924707195, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '533388', 'n_tokens_mistral': 3555, 'n_tokens_neox': 3466, 'n_words': 1001}