<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/bitly/go-simplejson"
	"io"
	"net/http"
	"os"
)

func usage() {
	os.Stderr.WriteString("You must pass the location of the files.\n" +
		"Usage: " + os.Args[0] + " /path/to/com.nubee.valkyriecrusade/files\n")
}

func main() {
	if len(os.Args) == 1 {
		usage()
		return
	}

	if _, err := os.Stat(os.Args[1]); os.IsNotExist(err) {
		usage()
		return
	}

	http.HandleFunc("/", masterDataHandler)
	http.ListenAndServe(":8080", nil)
}

func masterDataHandler(w http.ResponseWriter, r *http.Request) {
	// 0. decode all the files (or maybe just decode on demand?)
	// 1. load in json data from masterdata.dat
	// 2. list main keys as navigation points

	// File header
	io.WriteString(w, "<html><body>\n")

	// Step 0.
	filename := os.Args[1] + "/response/master_all.dat"
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		// here we can check if the non converted file exists and convert it
		fmt.Fprintf(w, "no such file or directory: %s", filename)
		io.WriteString(w, "</body></html>")
		return
	}

	f, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(w, "Error opening: %s", filename)
		io.WriteString(w, "</body></html>")
		return
	}

	masterData, _ := simplejson.NewFromReader(bufio.NewReader(f))

	masterDataMap, _ := masterData.Map()

	for k, _ := range masterDataMap {
		io.WriteString(w, k)
		io.WriteString(w, "<br />\n")
	}

	io.WriteString(w, "</body></html>")
}<commit_msg>sorts the keys<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/bitly/go-simplejson"
	"io"
	"net/http"
	"os"
	"sort"
	// "strings"
)

func usage() {
	os.Stderr.WriteString("You must pass the location of the files.\n" +
		"Usage: " + os.Args[0] + " /path/to/com.nubee.valkyriecrusade/files\n")
}

func main() {
	if len(os.Args) == 1 {
		usage()
		return
	}

	if _, err := os.Stat(os.Args[1]); os.IsNotExist(err) {
		usage()
		return
	}

	http.HandleFunc("/", masterDataHandler)
	http.ListenAndServe(":8080", nil)
}

func masterDataHandler(w http.ResponseWriter, r *http.Request) {
	// 0. decode all the files (or maybe just decode on demand?)
	// 1. load in json data from masterdata.dat
	// 2. list main keys as navigation points

	// File header
	io.WriteString(w, "<html><body>\n")

	// Step 0.
	filename := os.Args[1] + "/response/master_all.dat"
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		// here we can check if the non converted file exists and convert it
		fmt.Fprintf(w, "no such file or directory: %s", filename)
		io.WriteString(w, "</body></html>")
		return
	}

	f, err := os.Open(filename)
	if err != nil {
		fmt.Fprintf(w, "Error opening: %s", filename)
		io.WriteString(w, "</body></html>")
		return
	}

	// read the decoded master data file
	masterData, _ := simplejson.NewFromReader(bufio.NewReader(f))

	masterDataMap, _ := masterData.Map()

	// sort the keys
	mk := make([]string, len(masterDataMap))
	i := 0
	var k string
	for k, _ = range masterDataMap {
		mk[i] = k
		i++
	}
	sort.Strings(mk)

	// print all the keys in order
	for _, k = range mk {
		io.WriteString(w, "<p><a href=\"")
		io.WriteString(w, k)
		io.WriteString(w, "\">")
		io.WriteString(w, k)
		io.WriteString(w, "</a><div style=\"padding-left:10px\">")
		// val, _ := masterData.Get(k).EncodePretty()
		// io.WriteString(w, strings.Replace(string(val), "\n", "<br />", -1))
		io.WriteString(w, "</div></p>\n")
	}

	io.WriteString(w, "</body></html>")
}
{'fraction_non_alphanumeric': 0.1826137689614936, 'fraction_numerical': 0.008168028004667444, 'mean_word_length': 3.4823529411764707, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5593941', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1318, 'n_words': 320}