<commit_before>// Package util provides utility functions for veyron tools.
//
// TODO(jsimsa): Create a repoutil package that hides different
// version control systems behind a single interface.
package util

import (
	"bufio"
	"bytes"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"veyron.io/lib/cmdline"
	"veyron.io/tools/lib/collect"
	"veyron.io/tools/lib/gitutil"
	"veyron.io/tools/lib/runutil"
)

// CL represents a changelist.
type CL struct {
	// Author identifies the author of the changelist.
	Author string
	// Email identifies the author's email.
	Email string
	// Description holds the description of the changelist.
	Description string
}

// Manifest represents a setting used for updating the veyron universe.
type Manifest struct {
	Imports  []Import  `xml:"imports>import"`
	Projects []Project `xml:"projects>project"`
	Tools    []Tool    `xml:"tools>tool"`
	XMLName  struct{}  `xml:"manifest"`
}

// Imports maps manifest import names to their detailed description.
type Imports map[string]Import

// Import represnts a manifest import.
type Import struct {
	// Name is the name under which the manifest can be found the
	// manifest repository.
	Name string `xml:"name,attr"`
}

// Projects maps veyron project names to their detailed description.
type Projects map[string]Project

// Project represents a veyron project.
type Project struct {
	// Exclude is flag used to exclude previously included projects.
	Exclude bool `xml:"exclude,attr"`
	// Name is the URL at which the project is hosted.
	Name string `xml:"name,attr"`
	// Path is the path used to store the project locally. Project
	// manifest uses paths that are relative to the VEYRON_ROOT
	// environment variable. When a manifest is parsed (e.g. in
	// RemoteProjects), the program logic converts the relative
	// paths to an absolute paths, using the current value of the
	// VEYRON_ROOT environment variable as a prefix.
	Path string `xml:"path,attr"`
	// Protocol is the version control protocol used by the
	// project. If not set, "git" is used as the default.
	Protocol string `xml:"protocol,attr"`
	// Revision is the revision the project should be advanced to
	// during "veyron update". If not set, "HEAD" is used as the
	// default.
	Revision string `xml:"revision,attr"`
}

// Tools maps veyron tool names, to their detailed description.
type Tools map[string]Tool

// Tool represents a veyron tool.
type Tool struct {
	// Exclude is flag used to exclude previously included projects.
	Exclude bool `xml:"exclude,attr"`
	// Name is the name of the tool binary.
	Name string `xml:"name,attr"`
	// Package is the package path of the tool.
	Package string `xml:"package,attr"`
	// Project identifies the project that contains the tool. If
	// not set, "https://veyron.googlesource.com/tools" is used as
	// the default.
	Project string `xml:"project,attr"`
}

type UnsupportedProtocolErr string

func (e UnsupportedProtocolErr) Error() string {
	return fmt.Sprintf("unsupported protocol %v", e)
}

// Update represents an update of veyron projects as a map from
// project names to a collections of commits.
type Update map[string][]CL

// CreateSnapshot creates a manifest that encodes the current state of
// master branches of all projects and writes this snapshot out to the
// given file.
func CreateSnapshot(ctx *Context, path string) error {
	// Create an in-memory representation of the build manifest.
	manifest, err := snapshotLocalProjects(ctx)
	if err != nil {
		return err
	}
	perm := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(filepath.Dir(path), perm); err != nil {
		return err
	}
	data, err := xml.MarshalIndent(manifest, "", "  ")
	if err != nil {
		return err
	}
	perm = os.FileMode(0644)
	if err := ctx.Run().WriteFile(path, data, perm); err != nil {
		return fmt.Errorf("WriteFile(%v, %v) failed: %v", path, err, perm)
	}
	return nil
}

// LocalProjects scans the local filesystem to identify existing
// projects.
func LocalProjects(ctx *Context) (Projects, error) {
	root, err := VeyronRoot()
	if err != nil {
		return nil, err
	}
	projects := Projects{}
	if err := findLocalProjects(ctx, root, projects); err != nil {
		return nil, err
	}
	return projects, nil
}

// PollProjects returns the set of changelists that exist remotely
// but not locally. Changes are grouped by veyron repositories and
// contain author identification and a description of their content.
func PollProjects(ctx *Context, manifest string, projectSet map[string]struct{}) (update Update, e error) {
	cwd, err := os.Getwd()
	if err != nil {<commit_msg>tools: fixing broken polling<commit_after>// Package util provides utility functions for veyron tools.
//
// TODO(jsimsa): Create a repoutil package that hides different
// version control systems behind a single interface.
package util

import (
	"bufio"
	"bytes"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"veyron.io/lib/cmdline"
	"veyron.io/tools/lib/collect"
	"veyron.io/tools/lib/gitutil"
	"veyron.io/tools/lib/runutil"
)

// CL represents a changelist.
type CL struct {
	// Author identifies the author of the changelist.
	Author string
	// Email identifies the author's email.
	Email string
	// Description holds the description of the changelist.
	Description string
}

// Manifest represents a setting used for updating the veyron universe.
type Manifest struct {
	Imports  []Import  `xml:"imports>import"`
	Projects []Project `xml:"projects>project"`
	Tools    []Tool    `xml:"tools>tool"`
	XMLName  struct{}  `xml:"manifest"`
}

// Imports maps manifest import names to their detailed description.
type Imports map[string]Import

// Import represnts a manifest import.
type Import struct {
	// Name is the name under which the manifest can be found the
	// manifest repository.
	Name string `xml:"name,attr"`
}

// Projects maps veyron project names to their detailed description.
type Projects map[string]Project

// Project represents a veyron project.
type Project struct {
	// Exclude is flag used to exclude previously included projects.
	Exclude bool `xml:"exclude,attr"`
	// Name is the URL at which the project is hosted.
	Name string `xml:"name,attr"`
	// Path is the path used to store the project locally. Project
	// manifest uses paths that are relative to the VEYRON_ROOT
	// environment variable. When a manifest is parsed (e.g. in
	// RemoteProjects), the program logic converts the relative
	// paths to an absolute paths, using the current value of the
	// VEYRON_ROOT environment variable as a prefix.
	Path string `xml:"path,attr"`
	// Protocol is the version control protocol used by the
	// project. If not set, "git" is used as the default.
	Protocol string `xml:"protocol,attr"`
	// Revision is the revision the project should be advanced to
	// during "veyron update". If not set, "HEAD" is used as the
	// default.
	Revision string `xml:"revision,attr"`
}

// Tools maps veyron tool names, to their detailed description.
type Tools map[string]Tool

// Tool represents a veyron tool.
type Tool struct {
	// Exclude is flag used to exclude previously included projects.
	Exclude bool `xml:"exclude,attr"`
	// Name is the name of the tool binary.
	Name string `xml:"name,attr"`
	// Package is the package path of the tool.
	Package string `xml:"package,attr"`
	// Project identifies the project that contains the tool. If
	// not set, "https://veyron.googlesource.com/tools" is used as
	// the default.
	Project string `xml:"project,attr"`
}

type UnsupportedProtocolErr string

func (e UnsupportedProtocolErr) Error() string {
	return fmt.Sprintf("unsupported protocol %v", e)
}

// Update represents an update of veyron projects as a map from
// project names to a collections of commits.
type Update map[string][]CL

// CreateSnapshot creates a manifest that encodes the current state of
// master branches of all projects and writes this snapshot out to the
// given file.
func CreateSnapshot(ctx *Context, path string) error {
	// Create an in-memory representation of the build manifest.
	manifest, err := snapshotLocalProjects(ctx)
	if err != nil {
		return err
	}
	perm := os.FileMode(0755)
	if err := ctx.Run().MkdirAll(filepath.Dir(path), perm); err != nil {
		return err
	}
	data, err := xml.MarshalIndent(manifest, "", "  ")
	if err != nil {
		return err
	}
	perm = os.FileMode(0644)
	if err := ctx.Run().WriteFile(path, data, perm); err != nil {
		return fmt.Errorf("WriteFile(%v, %v) failed: %v", path, err, perm)
	}
	return nil
}

// LocalProjects scans the local filesystem to identify existing
// projects.
func LocalProjects(ctx *Context) (Projects, error) {
	root, err := VeyronRoot()
	if err != nil {
		return nil, err
	}
	projects := Projects{}
	if err := findLocalProjects(ctx, root, projects); err != nil {
		return nil, err
	}
	return projects, nil
}

// PollProjects returns the set of changelists that exist remotely
// but not locally. Changes are grouped by veyron repositories and
// contain author identification and a description of their content.
func PollProjects(ctx *Context, manifest string, projectSet map[string]struct{}) (_ Update, e error) {
	update := Update{}
	cwd, err := os.Getwd()
	if err != nil {
{'fraction_non_alphanumeric': 0.10388623534520192, 'fraction_numerical': 0.0017368649587494573, 'mean_word_length': 3.8953241232731135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467701', 'n_tokens_mistral': 2787, 'n_tokens_neox': 2705, 'n_words': 1118}