<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)


/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { inf.Close(); return err; }

	/* Open the output file */
	outf, err := os.Create("stripped.txt");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { inf.Close(); outf.Close(); return err; }

	/* Create a scanner object to break this in to lines */
	scanner := bufio.NewScanner(inf)
	/* Declare a variable for the line */
	var line string;
	/* Loop over lines */
	for scanner.Scan() {
		/* Trim right space and then add the \n back on the end before writing */
		line = strings.TrimRight(scanner.Text(), " \t")+"\n"
		outf.Write([]byte(line));
	}

	/* Close all open files */
	inf.Close();
	outf.Close();

	/* No errors, so we return nil */
	return nil;
}

func WalkFunc(path string, info os.FileInfo, err error) error {
	/* list of directories to ignore */
	blacklist := []string{".bzr", ".cvs", ".git", ".hg", ".svn"}
	if contains(path, blacklist){
		fmt.Printf("Skipping version control dir: %s\n", path)
		return filepath.SkipDir
	} else {
		inf, err := os.Open(path)
		defer inf.Close();
		if (err!=nil) { inf.Close(); return err; }
		readStart := io.LimitReader(inf, 512);
		data, err := ioutil.ReadAll(readStart);
		fileType := http.DetectContentType(data);
		if strings.Contains(fileType, "text/plain"){
			fmt.Printf("Trimming: %v\n", path)
		} else {
			fmt.Printf("Skipping file of type '%v': %v\n", fileType, path)
		}
	}
	return nil<commit_msg>Write out copies for now and ignore directories themselves.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)


/* This function takes a string and returns
   a (potentially nil) error object */
func TTWS(filename string) error {
	/* Open the input file */
	inf, err := os.Open(filename);
	/* In case this function generates a "panic", be sure to close this file */
	defer inf.Close();
	/* Did we open it successfully?  If not, close and return. */
	if (err!=nil) { inf.Close(); return err; }

	/* Open the output file */
	outf, err := os.Create(filename+"1");
	/* In case this function generates a "panic", be sure to close this file */
	defer outf.Close();
	/* Did we open it succesfully?  If not, close all and return. */
	if (err!=nil) { inf.Close(); outf.Close(); return err; }

	/* Create a scanner object to break this in to lines */
	scanner := bufio.NewScanner(inf)
	/* Declare a variable for the line */
	var line string;
	/* Loop over lines */
	for scanner.Scan() {
		/* Trim right space and then add the \n back on the end before writing */
		line = strings.TrimRight(scanner.Text(), " \t")+"\n"
		outf.Write([]byte(line));
	}

	/* Close all open files */
	inf.Close();
	outf.Close();

	/* No errors, so we return nil */
	return nil;
}

func WalkFunc(path string, fi os.FileInfo, err error) error {
	/* list of directories to ignore */
	blacklist := []string{".bzr", ".cvs", ".git", ".hg", ".svn"}
	if contains(path, blacklist){
		fmt.Printf("Skipping version control dir: %s\n", path)
		return filepath.SkipDir
	} else {
		inf, err := os.Open(path)
		defer inf.Close();
		if (err!=nil) { inf.Close(); return err; }
		readStart := io.LimitReader(inf, 512);
		data, err := ioutil.ReadAll(readStart);
		/* Close all open files */
		inf.Close();
		/* Determine file type */
		fileType := http.DetectContentType(data);
		if (strings.Contains(fileType, "text/plain") && !fi.IsDir()){
			fmt.Printf("Trimming: %v\n", path);
			TTWS(path);
		} else if !fi.IsDir() {
			fmt.Printf("Skipping file of type '%v': %v\n", fileType, path)
		}
	}
	return nil
{'fraction_non_alphanumeric': 0.1621160409556314, 'fraction_numerical': 0.0017064846416382253, 'mean_word_length': 3.4308855291576674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1141658', 'n_tokens_mistral': 1488, 'n_tokens_neox': 1417, 'n_words': 475}