<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/src-d/simple-linguist.v1"
)

func main() {
	flag.Usage = usage
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	errors := false
	o := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errors = true
			log.Println(err)
			return filepath.SkipDir
		}

		if slinguist.IsVendor(f.Name()) || slinguist.IsDotFile(f.Name()) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		l := slinguist.GetLanguage(filepath.Base(path), content)

		r, err := filepath.Rel(root, path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		o[l] = append(o[l], r)
		return nil
	})

	if err != nil {
		log.Fatal(err)
	}

	js, _ := json.MarshalIndent(o, "", "  ")
	fmt.Printf("%s\n", js)
<commit_msg>fixed cli to skip some type of directories and files<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"gopkg.in/src-d/simple-linguist.v1"
)

func main() {
	flag.Usage = usage
	flag.Parse()
	root, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	errors := false
	out := make(map[string][]string, 0)
	err = filepath.Walk(root, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			errors = true
			log.Println(err)
			return filepath.SkipDir
		}

		relativePath, err := filepath.Rel(root, path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		if relativePath == "." {
			return nil
		}

		if f.IsDir() {
			relativePath = relativePath + "/"
		}

		if slinguist.IsVendor(relativePath) || slinguist.IsDotFile(relativePath) ||
			slinguist.IsDocumentation(relativePath) || slinguist.IsConfiguration(relativePath) {
			if f.IsDir() {
				return filepath.SkipDir
			}

			return nil
		}

		if f.IsDir() {
			return nil
		}

		content, err := ioutil.ReadFile(path)
		if err != nil {
			errors = true
			log.Println(err)
			return nil
		}

		language := slinguist.GetLanguage(filepath.Base(path), content)
		if language == slinguist.OtherLanguage {
			return nil
		}

		out[language] = append(out[language], relativePath)
		return nil
	})

	if err != nil {
		log.Fatal(err)
	}

	data, _ := json.MarshalIndent(out, "", "  ")
	fmt.Printf("%s\n", data)

{'fraction_non_alphanumeric': 0.15349016583108369, 'fraction_numerical': 0.0023139220979560356, 'mean_word_length': 3.021705426356589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6113335', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1022, 'n_words': 218}