<commit_before>package main

import (
    "iterator"
    "fmt"
    "mapreduce"
    "os"
    "regexp"
)


func find_files(dirname string) chan interface{} {
    output := make(chan interface{})

    go func() {
        _find_files(dirname, output)
        close(output)
    }()

    return output
}


func _find_files(dirname string, output chan interface{}) {
    dir, _ := os.Open(dirname)
    dirnames, _ := dir.Readdirnames(-1)

    for i := 0; i < len(dirnames); i++ {
        fullpath := dirname + "/" + dirnames[i]
        file, _ := os.Stat(fullpath)

        if file.IsDir() {
            _find_files(fullpath, output)
        } 

        else {
            output <- fullpath
        }
    }
}


func word_count(filename interface{}, output chan interface{}) {
    results := map[string]int{}
    words := regexp.MustCompile(`[A-Za-z0-9_]*`)

    for line := range iterator.line(filename.(string)) {
        for _, match := range words.FindAllString(line, -1) {
            results[match]++
        }
    }

    output <- results
}


func reducer(input chan interface{}, output chan interface{}) {
    results := map[string]int{}

    for new_matches := range input {
        for key, value := range new_matches.(map[string]int) {
            previous_count, exists := results[key]

            if !exists {
                results[key] = value
            } else {
                results[key] = previous_count + value
            }
        }
    }

    output <- results
}


func main() {
    fmt.Print(mapreduce.MapReduce(word_count, reducer, find_files("."), 20))
}<commit_msg>Update wordcount.go<commit_after>package main

import (
    "iterator"
    "fmt"
    "mapreduce"
    "os"
    "regexp"
)

func find_files(dirname string) chan interface{} {
    output := make(chan interface{})

    go func() {
        _find_files(dirname, output)
        close(output)
    }()

    return output
}

func _find_files(dirname string, output chan interface{}) {
    dir, _ := os.Open(dirname)
    dirnames, _ := dir.Readdirnames(-1)

    for i := 0; i < len(dirnames); i++ {
        fullpath := dirname + "/" + dirnames[i]
        file, _ := os.Stat(fullpath)

        if file.IsDir() {
            _find_files(fullpath, output)
        } else {
            output <- fullpath
        }
    }
}

func word_count(filename interface{}, output chan interface{}) {
    results := map[string]int{}
    words := regexp.MustCompile(`[A-Za-z0-9_]*`)

    for line := range iterator.line(filename.(string)) {
        for _, match := range words.FindAllString(line, -1) {
            results[match]++
        }
    }

    output <- results
}

func reducer(input chan interface{}, output chan interface{}) {
    results := map[string]int{}

    for new_matches := range input {
        for key, value := range new_matches.(map[string]int) {
            previous_count, exists := results[key]

            if !exists {
                results[key] = value
            } else {
                results[key] = previous_count + value
            }
        }
    }

    output <- results
}

func main() {
    fmt.Print(mapreduce.MapReduce(word_count, reducer, find_files("."), 20))
}
{'fraction_non_alphanumeric': 0.13654998423210343, 'fraction_numerical': 0.004415011037527594, 'mean_word_length': 2.1098039215686275, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2829289', 'n_tokens_mistral': 1020, 'n_tokens_neox': 961, 'n_words': 221}