<commit_before>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"io/ioutil"
	"path/filepath"
)

func getFiles(directory string) []*dataaccess.File {

	childs := make([]*dataaccess.File, 0)

	filesDirectoryEntries, err := ioutil.ReadDir(directory)
	if err != nil {
		return childs
	}

	for _, directoryEntry := range filesDirectoryEntries {

		// append new file
		path := filepath.Join(directory, directoryEntry.Name())
		file, err := newFile(path)
		if err != nil {
			fmt.Printf("Unable to add file %q to index.\nError: %s\n", path, err)
		}

		childs = append(childs, file)
	}

	return childs
}

func newFile(path string) (*dataaccess.File, error) {

	// check if the path is a file
	if isFile, _ := fsutil.IsFile(path); !isFile {
		return nil, fmt.Errorf("%q is not a file.", path)
	}

	// route
	route, err := route.New(path)
	if err != nil {
		return nil, fmt.Errorf("Cannot create a File for the path %q. Error: %s", path, err)
	}

	// hash provider
	hashProvider := func() (string, error) {
<commit_msg>Fixed the getFiles method of the dataaccess package. The method did not check for directories before and did not recurse.<commit_after>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"io/ioutil"
	"path/filepath"
)

func getFiles(directory string) []*dataaccess.File {

	childs := make([]*dataaccess.File, 0)

	filesDirectoryEntries, err := ioutil.ReadDir(directory)
	if err != nil {
		return childs
	}

	for _, directoryEntry := range filesDirectoryEntries {

		path := filepath.Join(directory, directoryEntry.Name())

		// recurse if the path is a directory
		if isDir, _ := fsutil.IsDirectory(path); isDir {
			childs = append(childs, getFiles(path)...)
			continue
		}

		// append new file
		file, err := newFile(path)
		if err != nil {
			fmt.Printf("Unable to add file %q to index.\nError: %s\n", path, err)
		}

		childs = append(childs, file)
	}

	return childs
}

func newFile(path string) (*dataaccess.File, error) {

	// check if the path is a file
	if isFile, _ := fsutil.IsFile(path); !isFile {
		return nil, fmt.Errorf("%q is not a file.", path)
	}

	// route
	route, err := route.New(path)
	if err != nil {
		return nil, fmt.Errorf("Cannot create a File for the path %q. Error: %s", path, err)
	}

	// hash provider
	hashProvider := func() (string, error) {

{'fraction_non_alphanumeric': 0.13212163579168124, 'fraction_numerical': 0.005592450192240475, 'mean_word_length': 3.66884176182708, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3490488', 'n_tokens_mistral': 1030, 'n_tokens_neox': 969, 'n_words': 298}