<commit_before>package fdcount

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type Counter struct {
	match         string
	startingCount int
	startingOut   []byte
}

// Returns a count of the file descriptors matching the given string (not a
// regex). Also returns a Counter that can be used to check the delta of file
// descriptors after this point.
//
// Currently only works on systems that have the lsof command available.
//
// see https://groups.google.com/forum/#!topic/golang-nuts/c0AnWXjzNIA
//
func Matching(s string) (int, *Counter, error) {
	c := &Counter{match: s}

	// Count initial file descriptors
	out, err := runLsof()
	if err != nil {
		return 0, nil, err
	}
	c.startingOut = out
	c.startingCount = c.countMatches(out)
	return c.startingCount, c, nil
}

// Asserts that the number of file descriptors added/removed since Counter was
// created euqlas the given number.
func (c *Counter) AssertDelta(expected int) error {
	out, err := runLsof()
	if err != nil {
		return err
	}
	actual := c.countMatches(out) - c.startingCount
	if actual != expected {
		return fmt.Errorf("Unexpected TCP file descriptor count. Expected %d, have %d.\n\n%s",
			expected, actual, lsofDelta(string(c.startingOut), string(out)))
	}
	return nil
}

func (c *Counter) countMatches(out []byte) int {
	return bytes.Count(out, []byte(c.match))
}

func runLsof() ([]byte, error) {
	out, err := exec.Command("lsof", "-p", fmt.Sprintf("%v", os.Getpid())).Output()
	if err != nil {
		err = fmt.Errorf("Unable to run lsof: %v", err)
	}
	return out, err
}

func lsofDelta(start string, end string) string {
	startLines := strings.Split(start, "\n")
	endLines := strings.Split(end, "\n")

	added := make(map[string]interface{})
	removed := make(map[string]interface{})

	for _, line := range startLines {
		removed[line] = nil
	}<commit_msg>Better file descriptor diff reporting<commit_after>package fdcount

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type Counter struct {
	match         string
	startingLines string
	startingCount int
}

// Returns a count of the file descriptors matching the given string (not a
// regex). Also returns a Counter that can be used to check the delta of file
// descriptors after this point.
//
// Currently only works on systems that have the lsof command available.
//
// see https://groups.google.com/forum/#!topic/golang-nuts/c0AnWXjzNIA
//
func Matching(s string) (int, *Counter, error) {
	c := &Counter{match: s}

	// Count initial file descriptors
	out, err := runLsof()
	if err != nil {
		return 0, nil, err
	}
	c.startingLines, c.startingCount = c.matchingLines(out)
	return c.startingCount, c, nil
}

// Asserts that the number of file descriptors added/removed since Counter was
// created euqlas the given number.
func (c *Counter) AssertDelta(expected int) error {
	out, err := runLsof()
	if err != nil {
		return err
	}
	endingLines, endingCount := c.matchingLines(out)
	actual := endingCount - c.startingCount
	if actual != expected {
		return fmt.Errorf("Unexpected TCP file descriptor count. Expected %d, have %d.\n\n%s",
			expected, actual, lsofDelta(c.startingLines, endingLines))
	}
	return nil
}

func (c *Counter) matchingLines(out []byte) (string, int) {
	lines := make([]string, 0)
	for _, line := range strings.Split(string(out), "\n") {
		if strings.Contains(line, c.match) {
			lines = append(lines, line)
		}
	}
	return strings.Join(lines, "\n"), len(lines)
}

func (c *Counter) countMatches(out []byte) int {
	return bytes.Count(out, []byte(c.match))
}

func runLsof() ([]byte, error) {
	out, err := exec.Command("lsof", "-p", fmt.Sprintf("%v", os.Getpid())).Output()
	if err != nil {
		err = fmt.Errorf("Unable to run lsof: %v", err)
	}
	return out, err
}

func lsofDelta(start string, end string) string {
	startLines := strings.Split(start, "\n")
	endLines := strings.Split(end, "\n")

	added := make(map[string]interface{})
	removed := make(map[string]interface{})

	for _, line := range startLines {
		removed[line] = nil
	}
{'fraction_non_alphanumeric': 0.14310689310689312, 'fraction_numerical': 0.0012487512487512488, 'mean_word_length': 3.8841463414634148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4659928', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1349, 'n_words': 429}