<commit_before>package gitmedia

import (
	_ "../gitconfig"
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

type PathCommand struct {
	*Command
}

func (c *PathCommand) Run() {
	var sub string
	if len(c.SubCommands) > 0 {
		sub = c.SubCommands[0]
	}

	switch sub {
	case "add":
		c.addPath()
	case "remove":
		c.removePath()
	default:
		c.listPaths()
	}

}

func (c *PathCommand) addPath() {
	if len(c.SubCommands) < 2 {
		fmt.Println("git media path add <path> [path]*")
		return
	}

	knownPaths := findPaths()
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		fmt.Println("Error opening .gitattributes file")
		return
	}

	for _, t := range c.SubCommands[1:] {
		isKnownPath := false
		for _, k := range knownPaths {
			if t == k.Path {
				isKnownPath = true
			}
		}

		if isKnownPath {
			fmt.Println(t, "already supported")
			continue
		}

		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=media -crlf\n", t))
		if err != nil {
			fmt.Println("Error adding path", t)
			continue
		}
		fmt.Println("Adding path", t)
	}

	attributesFile.Close()
}

func (c *PathCommand) removePath() {
	if len(c.SubCommands) < 2 {
		fmt.Println("git meda path remove <path> [path]*")
		return
	}

	data, err := ioutil.ReadFile(".gitattributes")
	if err != nil {
		return
	}

	attributes := strings.NewReader(string(data))

	attributesFile, err := os.Create(".gitattributes")
	if err != nil {
		fmt.Println("Error opening .gitattributes for writing")
		return
	}

	scanner := bufio.NewScanner(attributes)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "filter=media") {
			fields := strings.Fields(line)
			removeThisPath := false
			for _, t := range c.SubCommands[1:] {
				if t == fields[0] {
					removeThisPath = true
				}
			}

			if !removeThisPath {
				attributesFile.WriteString(line + "\n")
			} else {
				fmt.Println("Removing path", fields[0])
			}
		}
	}

	attributesFile.Close()
}

func (c *PathCommand) listPaths() {
	fmt.Println("Listing paths")
	knownPaths := findPaths()
	for _, t := range knownPaths {
		fmt.Printf("    %s (%s)\n", t.Path, t.Source)
	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)

	attributes, err := os.Open(".gitattributes")
	if err != nil {
		return paths // No .gitattibtues == no file paths
	}

	scanner := bufio.NewScanner(attributes)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}

		if strings.Contains(line, "filter=media") {
			fields := strings.Fields(line)
			paths = append(paths, mediaPath{Path: fields[0], Source: ".gitattributes"})
		}
	}

	return paths
}

func init() {<commit_msg>Walk the project looking for .gitattributes files when listing paths<commit_after>package gitmedia

import (
	_ "../gitconfig"
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type PathCommand struct {
	*Command
}

func (c *PathCommand) Run() {
	var sub string
	if len(c.SubCommands) > 0 {
		sub = c.SubCommands[0]
	}

	switch sub {
	case "add":
		c.addPath()
	case "remove":
		c.removePath()
	default:
		c.listPaths()
	}

}

func (c *PathCommand) addPath() {
	if len(c.SubCommands) < 2 {
		fmt.Println("git media path add <path> [path]*")
		return
	}

	knownPaths := findPaths()
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		fmt.Println("Error opening .gitattributes file")
		return
	}

	for _, t := range c.SubCommands[1:] {
		isKnownPath := false
		for _, k := range knownPaths {
			if t == k.Path {
				isKnownPath = true
			}
		}

		if isKnownPath {
			fmt.Println(t, "already supported")
			continue
		}

		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=media -crlf\n", t))
		if err != nil {
			fmt.Println("Error adding path", t)
			continue
		}
		fmt.Println("Adding path", t)
	}

	attributesFile.Close()
}

func (c *PathCommand) removePath() {
	if len(c.SubCommands) < 2 {
		fmt.Println("git meda path remove <path> [path]*")
		return
	}

	data, err := ioutil.ReadFile(".gitattributes")
	if err != nil {
		return
	}

	attributes := strings.NewReader(string(data))

	attributesFile, err := os.Create(".gitattributes")
	if err != nil {
		fmt.Println("Error opening .gitattributes for writing")
		return
	}

	scanner := bufio.NewScanner(attributes)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "filter=media") {
			fields := strings.Fields(line)
			removeThisPath := false
			for _, t := range c.SubCommands[1:] {
				if t == fields[0] {
					removeThisPath = true
				}
			}

			if !removeThisPath {
				attributesFile.WriteString(line + "\n")
			} else {
				fmt.Println("Removing path", fields[0])
			}
		}
	}

	attributesFile.Close()
}

func (c *PathCommand) listPaths() {
	fmt.Println("Listing paths")
	knownPaths := findPaths()
	for _, t := range knownPaths {
		fmt.Printf("    %s (%s)\n", t.Path, t.Source)
	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findAttributeFiles() []string {
	paths := make([]string, 0)
	// TODO should find the project root, not just .
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})
	return paths
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			return paths
		}

		scanner := bufio.NewScanner(attributes)
		for scanner.Scan() {
			line := scanner.Text()
			if line == "" {
				continue
			}

			if strings.Contains(line, "filter=media") {
				fields := strings.Fields(line)
				paths = append(paths, mediaPath{Path: fields[0], Source: path})
			}
		}
	}

	return paths
}

func init() {
{'fraction_non_alphanumeric': 0.15461304494761743, 'fraction_numerical': 0.004900304156809733, 'mean_word_length': 3.345814977973568, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2849038', 'n_tokens_mistral': 2381, 'n_tokens_neox': 2132, 'n_words': 517}