<commit_before>//
// shorthand.go - command line utility to process shorthand definitions
// and render output with the transformed text and without any
// shorthand definitions.
//
// @author <NAME>, <<EMAIL>>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license.
// See: http://opensource.org/licenses/BSD-2-Clause
//
package main

import (
	shorthand "../../"
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

type expressionList []string

var (
	help       bool
	version    bool
	expression expressionList
	prompt     string
	noprompt   bool
	vm         *shorthand.VirtualMachine
	lineNo     int
)

var usage = func(exit_code int, msg string) {
	var fh = os.Stderr
	if exit_code == 0 {
		fh = os.Stdout
	}
	cmdName := os.Args[0]

	fmt.Fprintf(fh, `%s
USAGE %s [options] [FILES_TO_PROCESS]

%s is a command line utility to expand labels based on their
assigned definitions. The render output is the transformed text 
and without the shorthand definitions themselves. %s reads 
from standard input and writes to standard output. The basic 
definision form is

    LABEL := VALUE

When the label is encountered in the text outside of a definition
the label will be replaced with its value.

To create a shortand for the label "ACME" with the value
"the point at which someone or something is best"
would be done with the following line

    ACME := the point at which someone or something is best

Now each time the shorthand "ACME" is encountered the phrase
"the point at which someone or something is best" will replace it. Thus

    My, ACME, will come

would become

    My, the point at which someone or something is best, will come

These are supported types of assignments (followed by their forms). 

	+ Assign a string to a label

		LABEL := STRING

	+ Assign the contents of a file to a label

		LABEL :< FILENAME

	+ Assign the output of a Bash shell expression to a label

		LABEL :! BASH_EXPRESSION

	+ Assign the contents of a Shorthand expression to a label

		LABEL :{ SHORTHAND_EXPRESSION

	+ Read a file of shorthand assignments and assign any expansions to the label

		LABEL :={ SHORTHAND_FILENAME

	+ Write out the value for a label

		LABEL :> FILENAME

	+ Write out the values for all labels (order is not guaranteed)

		LABEL :=> FILENAME
		
	+ Write out the assignment statement for a label

		LABEL :} FILENAME

	+ Write out all the assignment statements for all labels

		LABEL :=} FILENAME

	+ Render markdown to a label

		LABEL :[ MARKDOWN_EXPRESSION

	+ Render markdown file to a label

		LABEL :=[ FILENAME


If the label is an underscore then it gets ignored.

You can evaluate shorthand expression in the command line much like you do
with the Unix command sed.  But you can also easily embed them in the file
or include from a file.

EXAMPLE

Pass the current date and time as shorthands transform the file "input.txt"
into "output.txt" with shorthands converted.

    %s -e "@now :! date +%%H:%%M" \
	   -e "@today :! date +%%Y-%%m-%%d" < input.txt > output.txt

You can also embed the shorthand definitions directly in your text file.

OPTIONS
`, msg, cmdName, cmdName, cmdName, cmdName)

	flag.VisitAll(func(f *flag.Flag) {
		fmt.Fprintf(fh, "\t-%s\t\t%s\n", f.Name, f.Usage)
	})

	fmt.Fprintf(fh, `
copyright (c) 2015 all rights reserved.
Released under the BSD 2-Clause license.
See: http://opensource.org/licenses/BSD-2-Clause
`)
	os.Exit(exit_code)
}

var welcome = `
  Welcome to shorthand the simple label expander and markdown processor.
  Use ':exit:' to quit the repl, ':help:' to get a list of supported operators.

`

var helpShorthand = func(vm *shorthand.VirtualMachine, sm shorthand.SourceMap) (shorthand.SourceMap, error) {
	fmt.Printf(`
The following operators are supported in shorthand:

`)
	for op, msg := range vm.Help {
		fmt.Printf("\t%s\t%s\n", op, msg)
	}
	return shorthand.SourceMap{Label: "", Op: ":help:", Source: "", Expanded: ""}, nil
}

//exitShorthand - call os.Exit() with appropriate value and exit the repl
var exitShorthand = func(vm *shorthand.VirtualMachine, sm shorthand.SourceMap) (shorthand.SourceMap, error) {
	if sm.Source == "" {
		os.Exit(0)
	}
	fmt.Fprintf(os.Stderr, sm.Source)
	os.Exit(1)
	return shorthand.SourceMap{Label: "", Op: ":exit:", Source: "", Expanded: ""}, nil<commit_msg>adding :help: to repl<commit_after>//
// shorthand.go - command line utility to process shorthand definitions
// and render output with the transformed text and without any
// shorthand definitions.
//
// @author <NAME>, <<EMAIL>>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license.
// See: http://opensource.org/licenses/BSD-2-Clause
//
package main

import (
	shorthand "../../"
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

type expressionList []string

var (
	help       bool
	version    bool
	expression expressionList
	prompt     string
	noprompt   bool
	vm         *shorthand.VirtualMachine
	lineNo     int
)

var usage = func(exit_code int, msg string) {
	var fh = os.Stderr
	if exit_code == 0 {
		fh = os.Stdout
	}
	cmdName := os.Args[0]

	fmt.Fprintf(fh, `%s
USAGE %s [options] [FILES_TO_PROCESS]

%s is a command line utility to expand labels based on their
assigned definitions. The render output is the transformed text 
and without the shorthand definitions themselves. %s reads 
from standard input and writes to standard output. The basic 
definision form is

    LABEL := VALUE

When the label is encountered in the text outside of a definition
the label will be replaced with its value.

To create a shortand for the label "ACME" with the value
"the point at which someone or something is best"
would be done with the following line

    ACME := the point at which someone or something is best

Now each time the shorthand "ACME" is encountered the phrase
"the point at which someone or something is best" will replace it. Thus

    My, ACME, will come

would become

    My, the point at which someone or something is best, will come

These are supported types of assignments (followed by their forms). 

	+ Assign a string to a label

		LABEL := STRING

	+ Assign the contents of a file to a label

		LABEL :< FILENAME

	+ Assign the output of a Bash shell expression to a label

		LABEL :! BASH_EXPRESSION

	+ Assign the contents of a Shorthand expression to a label

		LABEL :{ SHORTHAND_EXPRESSION

	+ Read a file of shorthand assignments and assign any expansions to the label

		LABEL :={ SHORTHAND_FILENAME

	+ Write out the value for a label

		LABEL :> FILENAME

	+ Write out the values for all labels (order is not guaranteed)

		LABEL :=> FILENAME
		
	+ Write out the assignment statement for a label

		LABEL :} FILENAME

	+ Write out all the assignment statements for all labels

		LABEL :=} FILENAME

	+ Render markdown to a label

		LABEL :[ MARKDOWN_EXPRESSION

	+ Render markdown file to a label

		LABEL :=[ FILENAME


If the label is an underscore then it gets ignored.

You can evaluate shorthand expression in the command line much like you do
with the Unix command sed.  But you can also easily embed them in the file
or include from a file.

EXAMPLE

Pass the current date and time as shorthands transform the file "input.txt"
into "output.txt" with shorthands converted.

    %s -e "@now :! date +%%H:%%M" \
	   -e "@today :! date +%%Y-%%m-%%d" < input.txt > output.txt

You can also embed the shorthand definitions directly in your text file.

OPTIONS
`, msg, cmdName, cmdName, cmdName, cmdName)

	flag.VisitAll(func(f *flag.Flag) {
		fmt.Fprintf(fh, "\t-%s\t\t%s\n", f.Name, f.Usage)
	})

	fmt.Fprintf(fh, `
copyright (c) 2015 all rights reserved.
Released under the BSD 2-Clause license.
See: http://opensource.org/licenses/BSD-2-Clause
`)
	os.Exit(exit_code)
}

var welcome = `
  Welcome to shorthand the simple label expander and markdown processor.
  Use ':exit:' to quit the repl, ':help:' to get a list of supported operators.

`

var helpShorthand = func(vm *shorthand.VirtualMachine, sm shorthand.SourceMap) (shorthand.SourceMap, error) {
	fmt.Printf(`
The following operators are supported in shorthand:

`)
	for op, msg := range vm.Help {
		fmt.Printf("\t%s\t%s\n", op, msg)
	}
	fmt.Printf("\nshorthand %s\n\n", shorthand.Version)
	return shorthand.SourceMap{Label: "", Op: ":help:", Source: "", Expanded: ""}, nil
}

//exitShorthand - call os.Exit() with appropriate value and exit the repl
var exitShorthand = func(vm *shorthand.VirtualMachine, sm shorthand.SourceMap) (shorthand.SourceMap, error) {
	if sm.Source == "" {
		os.Exit(0)
	}
	fmt.Fprintf(os.Stderr, sm.Source)
	os.Exit(1)
	return shorthand.SourceMap{Label: "", Op: ":exit:", Source: "", Expanded: ""}, nil
{'fraction_non_alphanumeric': 0.09357332410291912, 'fraction_numerical': 0.003692165685935156, 'mean_word_length': 3.6032926181625067, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3189306', 'n_tokens_mistral': 2879, 'n_tokens_neox': 2702, 'n_words': 1117}