<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"gop.kg/src-d/simple-linguist@v1"
)

func main() {
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	ifError(err)

	if root == "" {
		usage()
	}

	o := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if slinguist.IsVendor(f.Name()) || slinguist.IsDotFile(f.Name()) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		l, safe := slinguist.GetLanguageByExtension(path)
		if !safe {
			content, _ := ioutil.ReadFile(path)
			l, safe = slinguist.GetLanguageByContent(path, content)

		}

		r, _ := filepath.Rel(root, path)
		o[l] = append(o[l], r)
		return nil
	})

	ifError(err)

	js, _ := json.MarshalIndent(o, "", "  ")
	fmt.Printf("%s\n", js)
}

func usage() {
	fmt.Fprintf(
		os.Stderr, "simple-linguist, A simple (and faster) implementation of linguist \nusage: %s <path>\n",
		os.Args[0],
	)

	flag.PrintDefaults()
	os.Exit(2)
}

func ifError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(2)
	}
}
<commit_msg>update imports to srcd.works/simple-linguist.v1 (#11)<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"srcd.works/simple-linguist.v1"
)

func main() {
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	ifError(err)

	if root == "" {
		usage()
	}

	o := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if slinguist.IsVendor(f.Name()) || slinguist.IsDotFile(f.Name()) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		l, safe := slinguist.GetLanguageByExtension(path)
		if !safe {
			content, _ := ioutil.ReadFile(path)
			l, safe = slinguist.GetLanguageByContent(path, content)

		}

		r, _ := filepath.Rel(root, path)
		o[l] = append(o[l], r)
		return nil
	})

	ifError(err)

	js, _ := json.MarshalIndent(o, "", "  ")
	fmt.Printf("%s\n", js)
}

func usage() {
	fmt.Fprintf(
		os.Stderr, "simple-linguist, A simple (and faster) implementation of linguist \nusage: %s <path>\n",
		os.Args[0],
	)

	flag.PrintDefaults()
	os.Exit(2)
}

func ifError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(2)
	}
}

{'fraction_non_alphanumeric': 0.17625607779578606, 'fraction_numerical': 0.006077795786061589, 'mean_word_length': 3.2716262975778547, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6679074', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1041, 'n_words': 206}