<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"unicode/utf8"

	"appengine"
	"appengine/datastore"

	"cache"
	"key"
)

const commitsPerPage = 30

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		// Strip potentially large and unnecessary fields.
		com.ResultData = nil
		com.PerfResults = nil
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	if !isMasterKey(c, r.FormValue("key")) {
		return nil, errors.New("can only POST commits with master key")
	}

	// POST request
	body, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		return nil, fmt.Errorf("reading Body: %v", err)
	}
	if !bytes.Contains(body, needsBenchmarkingBytes) {
		c.Warningf("old builder detected at %v", r.RemoteAddr)
		return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
	}
	if err := json.Unmarshal(body, com); err != nil {
		return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
	}
	com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c appengine.Context) error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

var needsBenchmarkingBytes = []byte(`"NeedsBenchmarking"`)

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) error {
	var tc Commit // temp value so we don't clobber com
	err := datastore.Get(c, com.Key(c), &tc)
	if err != datastore.ErrNoSuchEntity {
		// if this commit is already in the datastore, do nothing
		if err == nil {
			return nil
		}
		return fmt.Errorf("getting Commit: %v", err)
	}
	// get the next commit number
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	com.Num = p.NextNum
	p.NextNum++
	if _, err := datastore.Put(c, p.Key(c), p); err != nil {
		return fmt.Errorf("putting Package: %v", err)
	}
	// if this isn't the first Commit test the parent commit exists
	if com.Num > 0 {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return errors.New("parent commit not found")
		}
	}
	// update the tip Tag if this is the Go repo and this isn't on a release branch
	if p.Path == "" && !strings.HasPrefix(com.Desc, "[release-branch") {
		t := &Tag{Kind: "tip", Hash: com.Hash}
		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
			return fmt.Errorf("putting Tag: %v", err)
		}<commit_msg>dashboard: more robust commit handler, and allow modifications<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"crypto/hmac"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"unicode/utf8"

	"appengine"
	"appengine/datastore"

	"cache"
	"key"
)

const commitsPerPage = 30

// commitHandler retrieves commit data or records a new commit.
//
// For GET requests it returns a Commit value for the specified
// packagePath and hash.
//
// For POST requests it reads a JSON-encoded Commit value from the request
// body and creates a new Commit entity. It also updates the "tip" Tag for
// each new commit at tip.
//
// This handler is used by a gobuilder process in -commit mode.
func commitHandler(r *http.Request) (interface{}, error) {
	c := contextForRequest(r)
	com := new(Commit)

	if r.Method == "GET" {
		com.PackagePath = r.FormValue("packagePath")
		com.Hash = r.FormValue("hash")
		if err := datastore.Get(c, com.Key(c), com); err != nil {
			return nil, fmt.Errorf("getting Commit: %v", err)
		}
		if com.Num == 0 {
			// Corrupt state which shouldn't happen but does.
			// Return an error so builders' commit loops will
			// be willing to retry submitting this commit.
			return nil, errors.New("in datastore with zero Num")
		}
		if com.Desc == "" || com.User == "" {
			// Also shouldn't happen, but at least happened
			// once on a single commit when trying to fix data
			// in the datastore viewer UI?
			return nil, errors.New("missing field")
		}
		// Strip potentially large and unnecessary fields.
		com.ResultData = nil
		com.PerfResults = nil
		return com, nil
	}
	if r.Method != "POST" {
		return nil, errBadMethod(r.Method)
	}
	if !isMasterKey(c, r.FormValue("key")) {
		return nil, errors.New("can only POST commits with master key")
	}

	// POST request
	body, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		return nil, fmt.Errorf("reading Body: %v", err)
	}
	if !bytes.Contains(body, needsBenchmarkingBytes) {
		c.Warningf("old builder detected at %v", r.RemoteAddr)
		return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
	}
	if err := json.Unmarshal(body, com); err != nil {
		return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
	}
	com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
	if err := com.Valid(); err != nil {
		return nil, fmt.Errorf("validating Commit: %v", err)
	}
	defer cache.Tick(c)
	tx := func(c appengine.Context) error {
		return addCommit(c, com)
	}
	return nil, datastore.RunInTransaction(c, tx, nil)
}

var needsBenchmarkingBytes = []byte(`"NeedsBenchmarking"`)

// addCommit adds the Commit entity to the datastore and updates the tip Tag.
// It must be run inside a datastore transaction.
func addCommit(c appengine.Context, com *Commit) error {
	var ec Commit // existing commit
	isUpdate := false
	err := datastore.Get(c, com.Key(c), &ec)
	if err != nil && err != datastore.ErrNoSuchEntity {
		return fmt.Errorf("getting Commit: %v", err)
	}
	if err == nil {
		// Commit already in the datastore. Any fields different?
		// If not, don't do anything.
		changes := (com.Num != 0 && com.Num != ec.Num) ||
			com.ParentHash != ec.ParentHash ||
			com.Desc != ec.Desc ||
			com.User != ec.User ||
			!com.Time.Equal(ec.Time)
		if !changes {
			return nil
		}
		ec.ParentHash = com.ParentHash
		ec.Desc = com.Desc
		ec.User = com.User
		if !com.Time.IsZero() {
			ec.Time = com.Time
		}
		if com.Num != 0 {
			ec.Num = com.Num
		}
		isUpdate = true
		com = &ec
	}
	p, err := GetPackage(c, com.PackagePath)
	if err != nil {
		return fmt.Errorf("GetPackage: %v", err)
	}
	if com.Num == 0 {
		// get the next commit number
		com.Num = p.NextNum
		p.NextNum++
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			return fmt.Errorf("putting Package: %v", err)
		}
	} else if com.Num >= p.NextNum {
		p.NextNum = com.Num + 1
		if _, err := datastore.Put(c, p.Key(c), p); err != nil {
			return fmt.Errorf("putting Package: %v", err)
		}
	}
	// if this isn't the first Commit test the parent commit exists.
	// The all zeros are returned by hg's p1node template for parentless commits.
	if com.ParentHash != "" && com.ParentHash != "0000000000000000000000000000000000000000" {
		n, err := datastore.NewQuery("Commit").
			Filter("Hash =", com.ParentHash).
			Ancestor(p.Key(c)).
			Count(c)
		if err != nil {
			return fmt.Errorf("testing for parent Commit: %v", err)
		}
		if n == 0 {
			return errors.New("parent commit not found")
		}
	}
	// update the tip Tag if this is the Go repo and this isn't on a release branch
	if p.Path == "" && !strings.HasPrefix(com.Desc, "[") && !isUpdate {
		t := &Tag{Kind: "tip", Hash: com.Hash}
		if _, err = datastore.Put(c, t.Key(c), t); err != nil {
			return fmt.Errorf("putting Tag: %v", err)
		}
{'fraction_non_alphanumeric': 0.14025229357798166, 'fraction_numerical': 0.007454128440366973, 'mean_word_length': 3.4023220595658756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4867272', 'n_tokens_mistral': 3244, 'n_tokens_neox': 3008, 'n_words': 998}