<commit_before>// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// export_to_sheets updates a Google sheets document with the latest test
// results
package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"./cause"
	"./consts"
	"./git"
	"./testlist"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/sheets/v4"
)

var (
	authdir       = flag.String("authdir", "~/.regres-auth", "directory to hold credentials.json and generated token")
	projectPath   = flag.String("projpath", ".", "project path")
	testListPath  = flag.String("testlist", "tests/regres/full-tests.json", "project relative path to the test list .json file")
	spreadsheetID = flag.String("spreadsheet", "1RCxbqtKNDG9rVMe_xHMapMBgzOCp24mumab73SbHtfw", "identifier of the spreadsheet to update")
)

const (
	columnGitHash = "GIT_HASH"
	columnGitDate = "GIT_DATE"
)

func main() {
	flag.Parse()

	if err := run(); err != nil {
		log.Fatalln(err)
	}
}

func run() error {
	// Load the full test list. We use this to find the test file names.
	lists, err := testlist.Load(".", *testListPath)
	if err != nil {
		return cause.Wrap(err, "Unable to load test list")
	}

	// Load the creditials used for editing the Google Sheets spreadsheet.
	srv, err := createSheetsService(*authdir)
	if err != nil {
		return cause.Wrap(err, "Unable to authenticate")
	}

	// Ensure that there is a sheet for each of the test lists.
	if err := createTestListSheets(srv, lists); err != nil {
		return cause.Wrap(err, "Unable to create sheets")
	}

	spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
	if err != nil {
		return cause.Wrap(err, "Unable to get spreadsheet")
	}

	req := sheets.BatchUpdateValuesRequest{
		ValueInputOption: "RAW",
	}

	testListDir := filepath.Dir(filepath.Join(*projectPath, *testListPath))
	changes, err := git.Log(testListDir, 100)
	if err != nil {
		return cause.Wrap(err, "Couldn't get git changes for '%v'", testListDir)
	}

	for _, group := range lists {
		sheetName := group.Name
		fmt.Println("Processing sheet", sheetName)
		sheet := getSheet(spreadsheet, sheetName)
		if sheet == nil {
			return cause.Wrap(err, "Sheet '%v' not found", sheetName)
		}

		columnHeaders, err := fetchRow(srv, spreadsheet, sheet, 0)
		if err != nil {
			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
		}

		columnIndices := listToMap(columnHeaders)

		hashColumnIndex, found := columnIndices[columnGitHash]
		if !found {
			return cause.Wrap(err, "Couldn't find sheet '%v' column header '%v'", sheetName, columnGitHash)
		}

		hashValues, err := fetchColumn(srv, spreadsheet, sheet, hashColumnIndex)
		if err != nil {
			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
		}
		hashValues = hashValues[1:] // Skip header

		hashIndices := listToMap(hashValues)
		rowValues := map[string]interface{}{}

		rowInsertionPoint := 1 + len(hashValues)

		for i := len(changes) - 1; i > 0; i-- {
			change := changes[i]
			if !strings.HasPrefix(change.Subject, consts.TestListUpdateCommitSubjectPrefix) {
				continue
			}

			hash := change.Hash.String()
			if _, found := hashIndices[hash]; found {
				continue // Already in the sheet
			}

			rowValues[columnGitHash] = change.Hash.String()
			rowValues[columnGitDate] = change.Date.Format("2006-01-02")<commit_msg>tests/regres/export_to_sheets: Fix off-by-one in loop<commit_after>// Copyright 2019 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// export_to_sheets updates a Google sheets document with the latest test
// results
package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"./cause"
	"./consts"
	"./git"
	"./testlist"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/sheets/v4"
)

var (
	authdir       = flag.String("authdir", "~/.regres-auth", "directory to hold credentials.json and generated token")
	projectPath   = flag.String("projpath", ".", "project path")
	testListPath  = flag.String("testlist", "tests/regres/full-tests.json", "project relative path to the test list .json file")
	spreadsheetID = flag.String("spreadsheet", "1RCxbqtKNDG9rVMe_xHMapMBgzOCp24mumab73SbHtfw", "identifier of the spreadsheet to update")
)

const (
	columnGitHash = "GIT_HASH"
	columnGitDate = "GIT_DATE"
)

func main() {
	flag.Parse()

	if err := run(); err != nil {
		log.Fatalln(err)
	}
}

func run() error {
	// Load the full test list. We use this to find the test file names.
	lists, err := testlist.Load(".", *testListPath)
	if err != nil {
		return cause.Wrap(err, "Unable to load test list")
	}

	// Load the creditials used for editing the Google Sheets spreadsheet.
	srv, err := createSheetsService(*authdir)
	if err != nil {
		return cause.Wrap(err, "Unable to authenticate")
	}

	// Ensure that there is a sheet for each of the test lists.
	if err := createTestListSheets(srv, lists); err != nil {
		return cause.Wrap(err, "Unable to create sheets")
	}

	spreadsheet, err := srv.Spreadsheets.Get(*spreadsheetID).Do()
	if err != nil {
		return cause.Wrap(err, "Unable to get spreadsheet")
	}

	req := sheets.BatchUpdateValuesRequest{
		ValueInputOption: "RAW",
	}

	testListDir := filepath.Dir(filepath.Join(*projectPath, *testListPath))
	changes, err := git.Log(testListDir, 100)
	if err != nil {
		return cause.Wrap(err, "Couldn't get git changes for '%v'", testListDir)
	}

	for _, group := range lists {
		sheetName := group.Name
		fmt.Println("Processing sheet", sheetName)
		sheet := getSheet(spreadsheet, sheetName)
		if sheet == nil {
			return cause.Wrap(err, "Sheet '%v' not found", sheetName)
		}

		columnHeaders, err := fetchRow(srv, spreadsheet, sheet, 0)
		if err != nil {
			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
		}

		columnIndices := listToMap(columnHeaders)

		hashColumnIndex, found := columnIndices[columnGitHash]
		if !found {
			return cause.Wrap(err, "Couldn't find sheet '%v' column header '%v'", sheetName, columnGitHash)
		}

		hashValues, err := fetchColumn(srv, spreadsheet, sheet, hashColumnIndex)
		if err != nil {
			return cause.Wrap(err, "Couldn't get sheet '%v' column headers", sheetName)
		}
		hashValues = hashValues[1:] // Skip header

		hashIndices := listToMap(hashValues)
		rowValues := map[string]interface{}{}

		rowInsertionPoint := 1 + len(hashValues)

		for i := len(changes) - 1; i >= 0; i-- {
			change := changes[i]
			if !strings.HasPrefix(change.Subject, consts.TestListUpdateCommitSubjectPrefix) {
				continue
			}

			hash := change.Hash.String()
			if _, found := hashIndices[hash]; found {
				continue // Already in the sheet
			}

			rowValues[columnGitHash] = change.Hash.String()
			rowValues[columnGitDate] = change.Date.Format("2006-01-02")
{'fraction_non_alphanumeric': 0.13213438241538653, 'fraction_numerical': 0.008242787560884226, 'mean_word_length': 3.9098712446351933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5695159', 'n_tokens_mistral': 2856, 'n_tokens_neox': 2609, 'n_words': 795}