<commit_before>//
//  sed.go
//  sed
//
// Copyright (c) 2009 <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package sed

import (
	"flag";
	"fmt";
	"io/ioutil";
	"os";
	"strings";
	"bytes";
	"container/vector";
)

const (
	versionMajor	= 0;
	versionMinor	= 1;
	versionPoint	= 0;
)

var versionString string

func init() {
	versionString = fmt.Sprintf("%d.%d.%d", versionMajor, versionMinor, versionPoint)
}

var show_version = flag.Bool("version", false, "Show version information.")
var show_help = flag.Bool("h", false, "Show help information.")
var quiet = flag.Bool("n", false, "Don't print the pattern space at the end of each script cycle.")
var script = flag.String("e", "", "The script used to process the input file.")
var script_file = flag.String("f", "", "Specify a file to read as the script. Ignored if -e present")
var edit_inplace = flag.Bool("i", false, "This option specifies that files are to be edited in-place. Otherwise output is printed to stdout.")
var line_wrap = flag.Uint("l", 70, "Specify the default line-wrap length for the l command. A length of 0 (zero) means to never wrap long lines. If not specified, it is taken to be 70.")
var unbuffered = flag.Bool("u", false, "Buffer both input and output as minimally as practical. (ignored)")
var treat_files_as_seperate = flag.Bool("s", false, "Treat files as searate entites. Line numbers reset to 1 for each file")

var usageShown bool = false

type Sed struct {
	inputLines		[][]byte;
	commands		*vector.Vector;
	outputFile		*os.File;
	patternSpace, holdSpace	[]byte;
	lineNumber		int;
}

func (s *Sed) Init() {
	s.commands = new(vector.Vector);
	s.outputFile = os.Stdout;
	s.patternSpace = make([]byte, 0);
	s.holdSpace = make([]byte, 0);
}

func usage() {
	// only show usage once.
	if !usageShown {
		usageShown = true;
		fmt.Fprint(os.Stdout, "sed [options] [script] input_file\n\n");
		flag.PrintDefaults();
	}
}

var inputFilename string

func (s *Sed) readInputFile() {
	b, err := ioutil.ReadFile(inputFilename);
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading input file %s\n", inputFilename);
		os.Exit(-1);
	}
	s.inputLines = bytes.Split(b, []byte{'\n'}, 0);
}

func (s *Sed) parseScript(scriptBuffer []byte) (err os.Error) {
	// a script may be a single command or it may be several
	scriptLines := bytes.Split(scriptBuffer, []byte{'\n'}, 0);
	for idx, line := range scriptLines {
		line = bytes.TrimSpace(line);
		if bytes.HasPrefix(line, []byte{'#'}) || len(line) == 0 {
			// comment
			continue
		}
		// this isn't really right. There may be slashes in the regular expression
		pieces := bytes.Split(line, []byte{'/'}, 0);
		c, err := NewCmd(pieces);
		if err != nil {
			fmt.Printf("Script error: %s -> %d: %s\n", err.String(), idx+1, line);
			os.Exit(-1);
		}
		s.commands.Push(c);
	}
	return nil;
}

func (s *Sed) printPatternSpace() {
	l := len(s.patternSpace);
	if *line_wrap <= 0 || l < int(*line_wrap) {
		fmt.Fprintf(s.outputFile, "%s\n", s.patternSpace)
	} else {
		// print the line in segments
		for i := 0; i < l; i += int(*line_wrap) {
			endOfLine := i + int(*line_wrap);
			if endOfLine > l {
				endOfLine = l
			}
			fmt.Fprintf(s.outputFile, "%s\n", s.patternSpace[i:endOfLine]);
		}
	}
}

func (s *Sed) process() {
	if *treat_files_as_seperate || *edit_inplace {
		s.lineNumber = 0
	}
	for _, s.patternSpace = range s.inputLines {
		// track line number starting with line 1
		s.lineNumber++;
		for c := range s.commands.Iter() {
			// println("cmd: ", c.(fmt.Stringer).String());
			if s.lineMatchesAddress(c.(Cmd).getAddress()) {
				stop, err := c.(Cmd).processLine(s);
				if err != nil {
					fmt.Printf("%v\n", err);
					os.Exit(-1);
				}
				if stop {
					break
				}
			}
		}
		if !*quiet {
			s.printPatternSpace()<commit_msg>Rather than indexing over range of input lines refactor so we can get lines from anyplace.<commit_after>//
//  sed.go
//  sed
//
// Copyright (c) 2009 <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package sed

import (
	"flag";
	"fmt";
	"io/ioutil";
	"os";
	"strings";
	"bytes";
	"container/vector";
)

const (
	versionMajor	= 0;
	versionMinor	= 1;
	versionPoint	= 0;
)

var versionString string

func init() {
	versionString = fmt.Sprintf("%d.%d.%d", versionMajor, versionMinor, versionPoint)
}

var show_version = flag.Bool("version", false, "Show version information.")
var show_help = flag.Bool("h", false, "Show help information.")
var quiet = flag.Bool("n", false, "Don't print the pattern space at the end of each script cycle.")
var script = flag.String("e", "", "The script used to process the input file.")
var script_file = flag.String("f", "", "Specify a file to read as the script. Ignored if -e present")
var edit_inplace = flag.Bool("i", false, "This option specifies that files are to be edited in-place. Otherwise output is printed to stdout.")
var line_wrap = flag.Uint("l", 70, "Specify the default line-wrap length for the l command. A length of 0 (zero) means to never wrap long lines. If not specified, it is taken to be 70.")
var unbuffered = flag.Bool("u", false, "Buffer both input and output as minimally as practical. (ignored)")
var treat_files_as_seperate = flag.Bool("s", false, "Treat files as searate entites. Line numbers reset to 1 for each file")

var usageShown bool = false

type Sed struct {
	inputLines		[][]byte;
	commands		*vector.Vector;
	outputFile		*os.File;
	patternSpace, holdSpace	[]byte;
	lineNumber		int;
}

func (s *Sed) Init() {
	s.commands = new(vector.Vector);
	s.outputFile = os.Stdout;
	s.patternSpace = make([]byte, 0);
	s.holdSpace = make([]byte, 0);
	s.lineNumber = 0;
}

func usage() {
	// only show usage once.
	if !usageShown {
		usageShown = true;
		fmt.Fprint(os.Stdout, "sed [options] [script] input_file\n\n");
		flag.PrintDefaults();
	}
}

var inputFilename string

func (s *Sed) readInputFile() {
	b, err := ioutil.ReadFile(inputFilename);
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading input file %s\n", inputFilename);
		os.Exit(-1);
	}
	s.inputLines = bytes.Split(b, []byte{'\n'}, 0);
}

func (s *Sed) parseScript(scriptBuffer []byte) (err os.Error) {
	// a script may be a single command or it may be several
	scriptLines := bytes.Split(scriptBuffer, []byte{'\n'}, 0);
	for idx, line := range scriptLines {
		line = bytes.TrimSpace(line);
		if bytes.HasPrefix(line, []byte{'#'}) || len(line) == 0 {
			// comment
			continue
		}
		// this isn't really right. There may be slashes in the regular expression
		pieces := bytes.Split(line, []byte{'/'}, 0);
		c, err := NewCmd(pieces);
		if err != nil {
			fmt.Printf("Script error: %s -> %d: %s\n", err.String(), idx+1, line);
			os.Exit(-1);
		}
		s.commands.Push(c);
	}
	return nil;
}

func (s *Sed) printPatternSpace() {
	l := len(s.patternSpace);
	if *line_wrap <= 0 || l < int(*line_wrap) {
		fmt.Fprintf(s.outputFile, "%s\n", s.patternSpace)
	} else {
		// print the line in segments
		for i := 0; i < l; i += int(*line_wrap) {
			endOfLine := i + int(*line_wrap);
			if endOfLine > l {
				endOfLine = l
			}
			fmt.Fprintf(s.outputFile, "%s\n", s.patternSpace[i:endOfLine]);
		}
	}
}

func (s *Sed)getNextLine()([]byte, os.Error) {
	if s.lineNumber < len(s.inputLines) {
		val := s.inputLines[s.lineNumber];
		s.lineNumber++;
		return val, nil;
	}
	return nil, os.EOF;
}

func (s *Sed) process() {
	if *treat_files_as_seperate || *edit_inplace {
		s.lineNumber = 0
	}
	var err os.Error;
	for s.patternSpace, err = s.getNextLine(); err == nil; s.patternSpace, err = s.getNextLine() {
		// track line number starting with line 1
		for c := range s.commands.Iter() {
			// println("cmd: ", c.(fmt.Stringer).String());
			if s.lineMatchesAddress(c.(Cmd).getAddress()) {
				stop, err := c.(Cmd).processLine(s);
				if err != nil {
					fmt.Printf("%v\n", err);
					os.Exit(-1);
				}
				if stop {
					break
				}
			}
		}
		if !*quiet {
			s.printPatternSpace()
{'fraction_non_alphanumeric': 0.14190821256038647, 'fraction_numerical': 0.005535426731078905, 'mean_word_length': 3.630475302889096, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '837906', 'n_tokens_mistral': 3531, 'n_tokens_neox': 3257, 'n_words': 1163}