<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aclements/go-misc/internal/loganal"
)

// TODO: If searching dashboard logs, optionally print to builder URLs
// instead of local file names.

// TODO: Optionally extract failures and show only those.

// TODO: Optionally classify matched logs by failure (and show either
// file name or extracted failure).

// TODO: Option to print Markdown-friendly output for GitHub.

// TODO: Option to print failure summary versus full failure message.

// TODO: Option to only show failures matching regexp? Currently we
// show all failures in files matching regexp, but sometimes you want
// to search the failures themselves. We could pre-filter the files by
// regexp, extract failures, and then match the failure messages.

var (
	// TODO: Allow mulitple -e's like grep.
	flagRegexp = flag.String("e", "", "show files matching `regexp`")
	re         *regexp.Regexp

	flagDashboard = flag.Bool("dashboard", false, "search dashboard logs from fetchlogs")
)

func main() {
	// XXX What I want right now is just to point it at a bunch of
	// logs and have it extract the failures.
	flag.Parse()

	// Validate flags.
	if *flagRegexp != "" {
		var err error
		re, err = regexp.Compile(*flagRegexp)
		if err != nil {
			fmt.Fprintf(os.Stderr, "bad regexp: %v\n", err)
			os.Exit(2)
		}
	}
	if *flagDashboard && flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
		os.Exit(2)
	}

	// Gather paths.
	var paths []string
	var stripDir string
	if *flagDashboard {
		revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
		paths = []string{revDir}
		stripDir = revDir + "/"
	} else {
		paths = flag.Args()
	}

	// Process files
	status := 1
	for _, path := range paths {
		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
				return nil
			}
			if info.IsDir() || strings.HasPrefix(filepath.Base(path), ".") {
				return nil
			}

			nicePath := path
			if stripDir != "" && strings.HasPrefix(path, stripDir) {
				nicePath = path[len(stripDir):]
			}

			found, err := process(path, nicePath)
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
			} else if found && status == 1 {
				status = 0
			}
			return nil
		})
	}
	os.Exit(status)
}

func process(path, nicePath string) (found bool, err error) {
	// TODO: Use streaming if possible.
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}

	// Check regexp match.
	if re != nil && !re.Match(data) {
		return false, nil
	}

	// Extract failures.
	failures, err := loganal.Extract(string(data), "", "")
	if err != nil {
		return false, err
	}

	// Print failures.
	for _, failure := range failures {
		msg := failure.FullMessage
		if msg == "" {
			msg = failure.Message
		}
		fmt.Printf("%s:\n%s\n\n", nicePath, msg)
	}
	return true, nil
}
<commit_msg>greplogs: compile regexp in multi-line mode<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aclements/go-misc/internal/loganal"
)

// TODO: If searching dashboard logs, optionally print to builder URLs
// instead of local file names.

// TODO: Optionally extract failures and show only those.

// TODO: Optionally classify matched logs by failure (and show either
// file name or extracted failure).

// TODO: Option to print Markdown-friendly output for GitHub.

// TODO: Option to print failure summary versus full failure message.

// TODO: Option to only show failures matching regexp? Currently we
// show all failures in files matching regexp, but sometimes you want
// to search the failures themselves. We could pre-filter the files by
// regexp, extract failures, and then match the failure messages.

var (
	// TODO: Allow mulitple -e's like grep.
	flagRegexp = flag.String("e", "", "show files matching `regexp`")
	re         *regexp.Regexp

	flagDashboard = flag.Bool("dashboard", false, "search dashboard logs from fetchlogs")
)

func main() {
	// XXX What I want right now is just to point it at a bunch of
	// logs and have it extract the failures.
	flag.Parse()

	// Validate flags.
	if *flagRegexp != "" {
		var err error
		re, err = regexp.Compile("(?m)" + *flagRegexp)
		if err != nil {
			fmt.Fprintf(os.Stderr, "bad regexp: %v\n", err)
			os.Exit(2)
		}
	}
	if *flagDashboard && flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "-dashboard and paths are incompatible\n")
		os.Exit(2)
	}

	// Gather paths.
	var paths []string
	var stripDir string
	if *flagDashboard {
		revDir := filepath.Join(xdgCacheDir(), "fetchlogs", "rev")
		paths = []string{revDir}
		stripDir = revDir + "/"
	} else {
		paths = flag.Args()
	}

	// Process files
	status := 1
	for _, path := range paths {
		filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
				return nil
			}
			if info.IsDir() || strings.HasPrefix(filepath.Base(path), ".") {
				return nil
			}

			nicePath := path
			if stripDir != "" && strings.HasPrefix(path, stripDir) {
				nicePath = path[len(stripDir):]
			}

			found, err := process(path, nicePath)
			if err != nil {
				status = 2
				fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
			} else if found && status == 1 {
				status = 0
			}
			return nil
		})
	}
	os.Exit(status)
}

func process(path, nicePath string) (found bool, err error) {
	// TODO: Use streaming if possible.
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}

	// Check regexp match.
	if re != nil && !re.Match(data) {
		return false, nil
	}

	// Extract failures.
	failures, err := loganal.Extract(string(data), "", "")
	if err != nil {
		return false, err
	}

	// Print failures.
	for _, failure := range failures {
		msg := failure.FullMessage
		if msg == "" {
			msg = failure.Message
		}
		fmt.Printf("%s:\n%s\n\n", nicePath, msg)
	}
	return true, nil
}

{'fraction_non_alphanumeric': 0.13318743160856653, 'fraction_numerical': 0.0037517586368610288, 'mean_word_length': 3.067387158296249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1627819', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2138, 'n_words': 728}