<commit_before>// Code for parsing coverage output in various formats.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"core"
)

// Parses test coverage for a single target from its output file.
func parseTestCoverage(target *core.BuildTarget, outputFile string) (core.TestCoverage, error) {
	coverage := core.NewTestCoverage()
	data, err := ioutil.ReadFile(outputFile)
	if err != nil && os.IsNotExist(err) {
		return coverage, nil // Tests aren't required to produce coverage files.
	} else if err != nil {
		return coverage, err
	} else if len(data) == 0 {
		return coverage, fmt.Errorf("Empty coverage output")
	} else if looksLikeGoCoverageResults(data) {
		// TODO(pebers): this is a little wasteful, we've already read the file once and we must do it again.
		return coverage, parseGoCoverageResults(target, &coverage, outputFile)
	} else {
		return coverage, parseXmlCoverageResults(target, &coverage, data)
	}
}

// Adds empty coverage entries for any files covered by the original query that we
// haven't discovered through tests to the overall report.
// The coverage reports only contain information about files that were covered during
// tests, so it's important that we identify anything with zero coverage here.
// This is made trickier by attempting to reconcile coverage targets from languages like
// Java that don't preserve the original file structure, which requires a slightly fuzzy match.
func AddOriginalTargetsToCoverage(state *core.BuildState) {
	// First we collect all the source files from all relevant targets
	allFiles := map[string]bool{}
	doneTargets := map[*core.BuildTarget]bool{}
	for _, label := range state.ExpandOriginalTargets() {
		collectAllFiles(state, state.Graph.TargetOrDie(label), allFiles, doneTargets)
	}

	// Now merge the recorded coverage so far into them
	recordedCoverage := state.Coverage
	state.Coverage = core.TestCoverage{Tests: recordedCoverage.Tests, Files: map[string][]core.LineCoverage{}}
	mergeCoverage(state, recordedCoverage, allFiles)
}

// Collects all the source files from a single target
func collectAllFiles(state *core.BuildState, target *core.BuildTarget, allFiles map[string]bool, doneTargets map[*core.BuildTarget]bool) {
	doneTargets[target] = true
	// Small hack here; explore these targets when we don't have any sources yet. Helps languages
	// like Java where we generate a wrapper target with a complete one immediately underneath.
	// TODO(pebers): do we still need this now we have Java sourcemaps?
	if !target.OutputIsComplete || len(allFiles) == 0 {
		for _, dep := range target.Dependencies() {
			if !doneTargets[dep] {
				collectAllFiles(state, dep, allFiles, doneTargets)
			}
		}
	}
	if target.IsTest {
		return // Test sources don't count for coverage.
	}
	for _, path := range target.AllSourcePaths(state.Graph) {
		extension := filepath.Ext(path)
		for _, ext := range state.Config.Cover.FileExtension {
			if ext == extension {
				allFiles[path] = target.IsTest || target.TestOnly // Skip test source files from actual coverage display
				break
			}
		}
	}
}

// Merges recorded coverage with the list of all existing files.
func mergeCoverage(state *core.BuildState, recordedCoverage core.TestCoverage, allFiles map[string]bool) {
	for file, coverage := range recordedCoverage.Files {
		state.Coverage.Files[file] = coverage
		allFiles[file] = true
	}
	// For any files left over now, enter them in as 100% uncovered.
	// This is pessimistic but there's not much we can do at this point.
	for file, done := range allFiles {
		if !done {
			s := make([]core.LineCoverage, countLines(file))
			if len(s) > 0 {
				for i := 0; i < len(s); i++ {
					s[i] = core.Uncovered
				}
				state.Coverage.Files[file] = s
			}
		}
	}
}

// Returns the number of lines in a file.
func countLines(path string) int {
	data, _ := ioutil.ReadFile(path)
	return bytes.Count(data, []byte{'\n'})<commit_msg>Filter packages for coverage to just the set of packages from all targets specified on the command line.<commit_after>// Code for parsing coverage output in various formats.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"core"
)

// Parses test coverage for a single target from its output file.
func parseTestCoverage(target *core.BuildTarget, outputFile string) (core.TestCoverage, error) {
	coverage := core.NewTestCoverage()
	data, err := ioutil.ReadFile(outputFile)
	if err != nil && os.IsNotExist(err) {
		return coverage, nil // Tests aren't required to produce coverage files.
	} else if err != nil {
		return coverage, err
	} else if len(data) == 0 {
		return coverage, fmt.Errorf("Empty coverage output")
	} else if looksLikeGoCoverageResults(data) {
		// TODO(pebers): this is a little wasteful, we've already read the file once and we must do it again.
		return coverage, parseGoCoverageResults(target, &coverage, outputFile)
	} else {
		return coverage, parseXmlCoverageResults(target, &coverage, data)
	}
}

// Adds empty coverage entries for any files covered by the original query that we
// haven't discovered through tests to the overall report.
// The coverage reports only contain information about files that were covered during
// tests, so it's important that we identify anything with zero coverage here.
// This is made trickier by attempting to reconcile coverage targets from languages like
// Java that don't preserve the original file structure, which requires a slightly fuzzy match.
func AddOriginalTargetsToCoverage(state *core.BuildState) {
	// First we collect all the source files from all relevant targets
	allFiles := map[string]bool{}
	doneTargets := map[*core.BuildTarget]bool{}
	// Track the set of packages the user ran tests from; we only show coverage metrics from them.
	coveragePackages := map[string]bool{}
	for _, label := range state.OriginalTargets {
		coveragePackages[label.PackageName] = true
	}
	for _, label := range state.ExpandOriginalTargets() {
		collectAllFiles(state, state.Graph.TargetOrDie(label), coveragePackages, allFiles, doneTargets)
	}

	// Now merge the recorded coverage so far into them
	recordedCoverage := state.Coverage
	state.Coverage = core.TestCoverage{Tests: recordedCoverage.Tests, Files: map[string][]core.LineCoverage{}}
	mergeCoverage(state, recordedCoverage, coveragePackages, allFiles)
}

// Collects all the source files from a single target
func collectAllFiles(state *core.BuildState, target *core.BuildTarget, coveragePackages, allFiles map[string]bool, doneTargets map[*core.BuildTarget]bool) {
	doneTargets[target] = true
	if !coveragePackages[target.Label.PackageName] {
		return
	}
	// Small hack here; explore these targets when we don't have any sources yet. Helps languages
	// like Java where we generate a wrapper target with a complete one immediately underneath.
	// TODO(pebers): do we still need this now we have Java sourcemaps?
	if !target.OutputIsComplete || len(allFiles) == 0 {
		for _, dep := range target.Dependencies() {
			if !doneTargets[dep] {
				collectAllFiles(state, dep, coveragePackages, allFiles, doneTargets)
			}
		}
	}
	if target.IsTest {
		return // Test sources don't count for coverage.
	}
	for _, path := range target.AllSourcePaths(state.Graph) {
		extension := filepath.Ext(path)
		for _, ext := range state.Config.Cover.FileExtension {
			if ext == extension {
				allFiles[path] = target.IsTest || target.TestOnly // Skip test source files from actual coverage display
				break
			}
		}
	}
}

// mergeCoverage merges recorded coverage with the list of all existing files.
func mergeCoverage(state *core.BuildState, recordedCoverage core.TestCoverage, coveragePackages, allFiles map[string]bool) {
	for file, coverage := range recordedCoverage.Files {
		if isOwnedBy(file, coveragePackages) {
			state.Coverage.Files[file] = coverage
			allFiles[file] = true
		}
	}
	// For any files left over now, enter them in as 100% uncovered.
	// This is pessimistic but there's not much we can do at this point.
	for file, done := range allFiles {
		if !done {
			s := make([]core.LineCoverage, countLines(file))
			if len(s) > 0 {
				for i := 0; i < len(s); i++ {
					s[i] = core.Uncovered
				}
				state.Coverage.Files[file] = s
			}
		}
	}
}

// isOwnedBy returns true if the given file is owned by any of the given packages.
func isOwnedBy(file string, coveragePackages map[string]bool) bool {
	for file != "." {
		file = path.Dir(file)
		if coveragePackages[file] {
			return true
		}
	}
	return false
}

// countLines returns the number of lines in a file.
func countLines(path string) int {
	data, _ := ioutil.ReadFile(path)
	return bytes.Count(data, []byte{'\n'})
{'fraction_non_alphanumeric': 0.09980475479499254, 'fraction_numerical': 0.0016079016882967728, 'mean_word_length': 4.1771700356718195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '312894', 'n_tokens_mistral': 2793, 'n_tokens_neox': 2570, 'n_words': 930}