<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// RunCommand runs cmd on file
func RunCommand(cmd string, path string) string {

	cmdOut, err := exec.Command(cmd, path).Output()
	assert(err)

	return string(cmdOut)
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) []byte {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	exifJSON := make(map[string]map[string]string)
	lines := strings.Split(exifout, "\n")
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !stringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(keyvalue[0])] = strings.TrimSpace(keyvalue[1])
		}
	}

	exifJSON["exiftool"] = datas
	jsonExif, err := json.Marshal(exifJSON)
	assert(err)

	return jsonExif
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) []byte {

	datas := make(map[string]string, 1)
	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")
	// Add hash to map
	datas["ssdeep"] = strings.TrimSpace(hashAndPath[0])

	jsonSsdeep, err := json.Marshal(datas)
	assert(err)

	return jsonSsdeep
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []byte {
	lines := strings.Split(tridout, "\n")
	lines = lines[5:]

	// fmt.Println(lines)

	datas := make(map[string][]string, 1)
	j, err := json.Marshal(datas)
	assert(err)

	return j
}<commit_msg>fix scan.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// RunCommand runs cmd on file
func RunCommand(cmd string, path string) string {

	cmdOut, err := exec.Command(cmd, path).Output()
	assert(err)

	return string(cmdOut)
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) []byte {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	exifJSON := make(map[string]map[string]string)
	lines := strings.Split(exifout, "\n")
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !stringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(keyvalue[0])] = strings.TrimSpace(keyvalue[1])
		}
	}

	exifJSON["exiftool"] = datas
	jsonExif, err := json.Marshal(exifJSON)
	assert(err)

	return jsonExif
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) []byte {

	datas := make(map[string]string, 1)
	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")
	// Add hash to map
	datas["ssdeep"] = strings.TrimSpace(hashAndPath[0])

	jsonSsdeep, err := json.Marshal(datas)
	assert(err)

	return jsonSsdeep
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []byte {
	lines := strings.Split(tridout, "\n")
	lines = lines[5:]

	// fmt.Println(lines)

	datas := make(map[string][]string, 1)
	datas["trid"] = lines
	j, err := json.Marshal(datas)
	assert(err)

	return j
}
{'fraction_non_alphanumeric': 0.14601195115614446, 'fraction_numerical': 0.004676539360872954, 'mean_word_length': 4.252387448840382, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '838997', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1385, 'n_words': 341}