<commit_before>package main

import (
	"bufio"
	"errors"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func extract(name string, config *Config) ([]*Dependency, []error) {
	fileinfo, err := os.Lstat(name)
	if err != nil {
		return nil, []error{err}
	}
	if fileinfo.IsDir() {
		if config.Recursive {
			var dependencies []*Dependency
			var errs []error
			filepath.Walk(name, func(name string, info os.FileInfo, err error) error {
				if err == nil && !info.IsDir() {
					deps, err := extractFile(name, config)
					errs = append(errs, err...)
					dependencies = append(dependencies, deps...)
				}
				return nil
			})
			return dependencies, errs
		} else {
			err := errors.New(name + " is a directory. Specify source code files. Or you mean --recursive (-r)?")
			return nil, []error{err}
		}
	} else {
		return extractFile(name, config)
	}
}

func extractFile(name string, config *Config) ([]*Dependency, []error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, []error{err}
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	module := path.Base(name)
	if config.Trimext {
		module = strings.TrimSuffix(module, filepath.Ext(module))
	}
	return extractCore(module, scanner, config), nil
}

func extractCore(module string, scanner *bufio.Scanner, config *Config) []*Dependency {
	dependency := &Dependency{From: module}
	dependencies := []*Dependency{dependency}
	appended := make(map[string]bool)
	enable := config.Start == nil
	for scanner.Scan() {
		line := scanner.Text()
		if config.Start != nil && config.Start.MatchString(line) {
			enable = true
		}
		if config.Module != nil {
			if matches := config.Module.FindStringSubmatch(line); matches != nil {
				dependency = &Dependency{From: matches[len(matches)-1]}
				dependencies = append(dependencies, dependency)
				appended = make(map[string]bool)
			}
		}
		if enable {
			if matches := config.Pattern.FindStringSubmatch(line); len(matches) >= 1 {
				for _, name := range matches[1:] {
					if name != "" && !appended[name] {
						dependency.To = append(dependency.To, name)
						appended[name] = true
					}
				}
			}
		}<commit_msg>improve extract.go: add to the list of dependencies only if there is one imported module at least<commit_after>package main

import (
	"bufio"
	"errors"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func extract(name string, config *Config) ([]*Dependency, []error) {
	fileinfo, err := os.Lstat(name)
	if err != nil {
		return nil, []error{err}
	}
	if fileinfo.IsDir() {
		if config.Recursive {
			var dependencies []*Dependency
			var errs []error
			filepath.Walk(name, func(name string, info os.FileInfo, err error) error {
				if err == nil && !info.IsDir() {
					deps, err := extractFile(name, config)
					errs = append(errs, err...)
					dependencies = append(dependencies, deps...)
				}
				return nil
			})
			return dependencies, errs
		} else {
			err := errors.New(name + " is a directory. Specify source code files. Or you mean --recursive (-r)?")
			return nil, []error{err}
		}
	} else {
		return extractFile(name, config)
	}
}

func extractFile(name string, config *Config) ([]*Dependency, []error) {
	file, err := os.Open(name)
	if err != nil {
		return nil, []error{err}
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	module := path.Base(name)
	if config.Trimext {
		module = strings.TrimSuffix(module, filepath.Ext(module))
	}
	return extractCore(module, scanner, config), nil
}

func extractCore(module string, scanner *bufio.Scanner, config *Config) []*Dependency {
	dependency := &Dependency{From: module}
	dependencies := []*Dependency{}
	appended := make(map[string]bool)
	enable := config.Start == nil
	for scanner.Scan() {
		line := scanner.Text()
		if config.Start != nil && config.Start.MatchString(line) {
			enable = true
		}
		if config.Module != nil {
			if matches := config.Module.FindStringSubmatch(line); matches != nil {
				dependency = &Dependency{From: matches[len(matches)-1]}
				appended = make(map[string]bool)
			}
		}
		if enable {
			if matches := config.Pattern.FindStringSubmatch(line); len(matches) >= 1 {
				for _, name := range matches[1:] {
					if name != "" && !appended[name] {
						if dependency.To == nil {
							dependencies = append(dependencies, dependency)
						}
						dependency.To = append(dependency.To, name)
						appended[name] = true
					}
				}
			}
		}
{'fraction_non_alphanumeric': 0.14907258987863523, 'fraction_numerical': 0.0013739409205404168, 'mean_word_length': 3.5405405405405403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5844399', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1438, 'n_words': 377}