<commit_before>package main
import (
    "fmt"
    "os"
    "strings"
    "bufio"
    "path/filepath"
    "strconv"
)

type dataset struct {
     title string
     path_to_file string
     json_data []map[string]interface{}
}

type uistate struct {
     phase int
     active_set *dataset
     key string
     value string
     datasets []dataset
     scanner *bufio.Scanner
}

func scan_or_exit(scanner *bufio.Scanner) {
    if scanner.Scan() == false && scanner.Err() == nil {
        os.Exit(0)
    }
}

func enter_interactive_loop(directory string) {
    scanner := bufio.NewScanner(os.Stdin)

    println("JSON Search Tool")

    datasets := find_datasets(directory)

    for {
        key, value, set := request_search_fields(datasets, scanner)
        if set != nil && set.json_data != nil {
            search_json(set.json_data, key, value)
        }
    }
    if err := scanner.Err(); err != nil {
        fmt.Println(os.Stderr, "error:", err)
        os.Exit(1)
    }
}

func find_datasets(directory string) []dataset {

    // Assumes no-one will be malicious enough to create a direcory with a '.json' suffix
    path_elements := []string { directory, "*.json" }
    files, _ := filepath.Glob(strings.Join(path_elements, "/"))

    datasets := make([]dataset, 0)

    for _, path_to_file := range files {
        elements := strings.Split(path_to_file, "/")
        filename := elements[len(elements)-1]
        set := dataset{strings.Title(strings.TrimSuffix(filename, ".json")), path_to_file, nil}
        datasets = append(datasets, set)
    }
    return datasets
}

func unpack_dataset(set *dataset) bool {
    if set.json_data == nil {
        valid, json_data := parse_file(set.path_to_file)
	if valid {
            (*set).json_data = json_data
	}
    }
    if set.json_data != nil {
        return true
    }    
    return false
}

func select_dataset(state uistate) uistate {
    fmt.Println("\nPlease select a dataset to search, or 'quit' to exit:")
    fmt.Printf("   ")
    for index, set := range state.datasets {
        if strings.Compare(set.title, "") != 0 {
            fmt.Printf("%v) %v ", index+1, set.title)
        }
    }
    fmt.Printf("\n# ")

    scan_or_exit(state.scanner)
    user_input := state.scanner.Text()
    if strings.Compare("quit", strings.ToLower(user_input)) == 0 {
        state.phase = -1
	return state
    }
    
    index, _ := strconv.ParseInt(user_input, 10, 32)
    if index > 0 && index <= int64(len(state.datasets)) {
        if unpack_dataset(&(state.datasets[index-1])) {
            state.active_set = &(state.datasets[index-1])
	    state.phase += 1
	    
        } else {
	    badset := state.datasets[index-1]
            fmt.Printf("Data source %v is corrupted. Please choose again.\n", badset.title)
            state.datasets[index-1].title = ""
        }

    } else {
        fmt.Printf("Invalid selection: '%v'.\n", user_input)
    }

    return state
}

func prompt_for_input(prompt string, help string) {
    fmt.Printf("\nEnter a %v to search for:\n", prompt)
    fmt.Printf("   '?' to see %v,\n", help)
    fmt.Printf("   '..' to go back\n")
    fmt.Printf("   'quit' to exit\n")
}

func select_field(state uistate) uistate {
    prompt_for_input("term", "available fields")
    fmt.Printf("%v # ", state.active_set.title)

    scan_or_exit(state.scanner)
    user_input := state.scanner.Text()

    if len(user_input) == 0 {
        // Could we ever wish to search all fields for a specific value?
        fmt.Printf("Invalid selection")
        
    } else if strings.Compare(user_input, "quit") == 0 {
        state.phase = -1

    } else if strings.Compare(user_input, "..") == 0 {
        state.phase -= 1

    } else if strings.Compare(user_input, "?") == 0 {
        if len(state.active_set.json_data) > 0 {
            // Assume for now that records are sufficiently uniform
            fmt.Printf("\n%v records contain the following fields\n", strings.TrimSuffix(state.active_set.title, "s"))
            for key, _:= range state.active_set.json_data[0] { 
                  fmt.Printf("* %s\n", key)
            }
        } else {
            fmt.Printf("* No records found *\n")
        }<commit_msg>Massage the user facing text<commit_after>package main
import (
    "fmt"
    "os"
    "strings"
    "bufio"
    "path/filepath"
    "strconv"
)

type dataset struct {
     title string
     path_to_file string
     json_data []map[string]interface{}
}

type uistate struct {
     phase int
     active_set *dataset
     key string
     value string
     datasets []dataset
     scanner *bufio.Scanner
}

func scan_or_exit(scanner *bufio.Scanner) {
    if scanner.Scan() == false && scanner.Err() == nil {
        os.Exit(0)
    }
}

func enter_interactive_loop(directory string) {
    scanner := bufio.NewScanner(os.Stdin)

    println("JSON Search Tool")

    datasets := find_datasets(directory)

    for {
        key, value, set := request_search_fields(datasets, scanner)
        if set != nil && set.json_data != nil {
            search_json(set.json_data, key, value)
        }
    }
    if err := scanner.Err(); err != nil {
        fmt.Println(os.Stderr, "error:", err)
        os.Exit(1)
    }
}

func find_datasets(directory string) []dataset {

    // Assumes no-one will be malicious enough to create a direcory with a '.json' suffix
    path_elements := []string { directory, "*.json" }
    files, _ := filepath.Glob(strings.Join(path_elements, "/"))

    datasets := make([]dataset, 0)

    for _, path_to_file := range files {
        elements := strings.Split(path_to_file, "/")
        filename := elements[len(elements)-1]
        set := dataset{strings.Title(strings.TrimSuffix(filename, ".json")), path_to_file, nil}
        datasets = append(datasets, set)
    }
    return datasets
}

func unpack_dataset(set *dataset) bool {
    if set.json_data == nil {
        valid, json_data := parse_file(set.path_to_file)
	if valid {
            (*set).json_data = json_data
	}
    }
    if set.json_data != nil {
        return true
    }    
    return false
}

func select_dataset(state uistate) uistate {
    fmt.Println("\nPlease select a dataset to search, or 'quit' to exit:")
    fmt.Printf("   ")
    for index, set := range state.datasets {
        if strings.Compare(set.title, "") != 0 {
            fmt.Printf("%v) %v ", index+1, set.title)
        }
    }
    fmt.Printf("\n# ")

    scan_or_exit(state.scanner)
    user_input := state.scanner.Text()
    if strings.Compare("quit", strings.ToLower(user_input)) == 0 {
        state.phase = -1
	return state
    }
    
    index, _ := strconv.ParseInt(user_input, 10, 32)
    if index > 0 && index <= int64(len(state.datasets)) {
        if unpack_dataset(&(state.datasets[index-1])) {
            state.active_set = &(state.datasets[index-1])
	    state.phase += 1
	    
        } else {
	    badset := state.datasets[index-1]
            fmt.Printf("Data source %v is corrupted. Please choose again.\n", badset.title)
            state.datasets[index-1].title = ""
        }

    } else {
        fmt.Printf("Invalid selection: '%v'.  Enter the number of a dataset instead.\n", user_input)
    }

    return state
}

func prompt_for_input(prompt string, help string) {
    fmt.Printf("\n   '?' to see %v\n", help)
    fmt.Printf("   '..' to go back\n")
    fmt.Printf("   'quit' to exit\n")
    fmt.Printf("\nEnter a %v to search for:\n", prompt)
}

func select_field(state uistate) uistate {
    prompt_for_input("term", "available fields")
    fmt.Printf("%v # ", state.active_set.title)

    scan_or_exit(state.scanner)
    user_input := state.scanner.Text()

    if len(user_input) == 0 {
        // Could we ever wish to search all fields for a specific value?
        fmt.Printf("Invalid selection")
        
    } else if strings.Compare(user_input, "quit") == 0 {
        state.phase = -1

    } else if strings.Compare(user_input, "..") == 0 {
        state.phase -= 1

    } else if strings.Compare(user_input, "?") == 0 {
        if len(state.active_set.json_data) > 0 {
            // Assume for now that records are sufficiently uniform
            fmt.Printf("\n%v records contain the following fields:\n", strings.TrimSuffix(state.active_set.title, "s"))
            for key, _:= range state.active_set.json_data[0] { 
                  fmt.Printf("* %s\n", key)
            }
        } else {
            fmt.Printf("* No records found *\n")
        }
{'fraction_non_alphanumeric': 0.13650906505510133, 'fraction_numerical': 0.006635857329067425, 'mean_word_length': 2.3746501399440225, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6727085', 'n_tokens_mistral': 2799, 'n_tokens_neox': 2729, 'n_words': 647}