<commit_before>package bundle

import (
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/getantibody/antibody/git"
)

var globs = []string{"*.plugin.zsh", "*.zsh", "*.sh", "*.zsh-theme"}

// Bundle is a git-based bundle/plugin
type Bundle struct {
	git.Repo
}

// New creates a new bundle instance
func New(fullName, folder string) Bundle {
	return Bundle{git.NewGithubRepo(fullName, folder)}
}

// Sourceables returns the list of files that could be sourced
func (b Bundle) Sourceables() []string {
	for _, glob := range globs {
		files, _ := filepath.Glob(filepath.Join(b.Folder(), glob))
		if files != nil {
			return files
		}
	}
	return nil
}

// List all bundles in the given folder
func List(folder string) []Bundle {
	entries, _ := ioutil.ReadDir(folder)
	var bundles []Bundle
	for _, bundle := range entries {
		if bundle.Mode().IsDir() && bundle.Name()[0] != '.' {
			bundles = append(bundles, New(bundle.Name(), folder))
		}
	}
	return bundles
}

// Parse a list of bundles, one per line, into a Bundle slice
func Parse(s, folder string) []Bundle {
	var bundles []Bundle
	for _, b := range strings.Split(s, "\n") {
		if strings.TrimSpace(b) != "" {
			bundles = append(bundles, New(b, folder))
		}
	}
	return bundles
}<commit_msg>strip comments when using stdin for bundles<commit_after>package bundle

import (
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/getantibody/antibody/git"
)

var globs = []string{"*.plugin.zsh", "*.zsh", "*.sh", "*.zsh-theme"}

// Bundle is a git-based bundle/plugin
type Bundle struct {
	git.Repo
}

// New creates a new bundle instance
func New(fullName, folder string) Bundle {
	return Bundle{git.NewGithubRepo(fullName, folder)}
}

// Sourceables returns the list of files that could be sourced
func (b Bundle) Sourceables() []string {
	for _, glob := range globs {
		files, _ := filepath.Glob(filepath.Join(b.Folder(), glob))
		if files != nil {
			return files
		}
	}
	return nil
}

// List all bundles in the given folder
func List(folder string) []Bundle {
	entries, _ := ioutil.ReadDir(folder)
	var bundles []Bundle
	for _, bundle := range entries {
		if bundle.Mode().IsDir() && bundle.Name()[0] != '.' {
			bundles = append(bundles, New(bundle.Name(), folder))
		}
	}
	return bundles
}

// Parse a list of bundles, one per line, into a Bundle slice
func Parse(s, folder string) []Bundle {
	var bundles []Bundle
	for _, decl := range strings.Split(s, "\n") {
		b := strings.Split(decl, "#")[0]
		b = strings.TrimSpace(b)
		if b != "" {
			bundles = append(bundles, New(b, folder))
		}
	}
	return bundles
}
{'fraction_non_alphanumeric': 0.1517509727626459, 'fraction_numerical': 0.0011673151750972762, 'mean_word_length': 3.7523105360443623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '59972', 'n_tokens_mistral': 941, 'n_tokens_neox': 880, 'n_words': 265}