<commit_before>// Copyright 2015 <NAME>. All Rights Reserved.
// Author: <EMAIL> (<NAME>)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to add missing fields to an existing GCS bucket, for #18.
//
// Input is of the form "<SHA-1> <CRC32C> <MD5>", e.g.:
//
//     e04b25d650dee1dff6ab1743724fa7c184282e94 0x12e9bf88 2bad5bb78f17232ef8c727f59eb82325
//
package main

import (
	"crypto/md5"
	"crypto/sha1"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

type crc32cChecksum uint32
type md5hash [md5.Size]byte
type sha1hash [sha1.Size]byte

type info struct {
	crc32c crc32cChecksum
	md5    md5hash
}

// List all blob objects in the GCS bucket into the channel.
func listBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	objects chan<- *gcs.Object) (err error)

// Filter to names of objects that lack the appropriate metadata keys.
func filterToProblematicNames(
	ctx context.Context,
	objects <-chan *gcs.Object,
	names chan<- string) (err error)

// For each object name, issue a request to set the appropriate metadata keys
// based on the contents of the supplied map. Write out the names of the
// objects processed.
func filterToProblematicNames(
	ctx context.Context,
	infoMap map[sha1hash]info,
	names <-chan string,
	processed chan<- string) (err error)
<commit_msg>Fixed a build error.<commit_after>// Copyright 2015 <NAME>. All Rights Reserved.
// Author: <EMAIL> (<NAME>)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to add missing fields to an existing GCS bucket, for #18.
//
// Input is of the form "<SHA-1> <CRC32C> <MD5>", e.g.:
//
//     e04b25d650dee1dff6ab1743724fa7c184282e94 0x12e9bf88 2bad5bb78f17232ef8c727f59eb82325
//
package main

import (
	"crypto/md5"
	"crypto/sha1"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

type crc32cChecksum uint32
type md5hash [md5.Size]byte
type sha1hash [sha1.Size]byte

type info struct {
	crc32c crc32cChecksum
	md5    md5hash
}

// List all blob objects in the GCS bucket into the channel.
func listBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	objects chan<- *gcs.Object) (err error)

// Filter to names of objects that lack the appropriate metadata keys.
func filterToProblematicNames(
	ctx context.Context,
	objects <-chan *gcs.Object,
	names chan<- string) (err error)

// For each object name, issue a request to set the appropriate metadata keys
// based on the contents of the supplied map. Write out the names of the
// objects processed.
func fixProblematicObjects(
	ctx context.Context,
	infoMap map[sha1hash]info,
	names <-chan string,
	processed chan<- string) (err error)

{'fraction_non_alphanumeric': 0.09692476661175178, 'fraction_numerical': 0.04503020318506315, 'mean_word_length': 3.7373211963589075, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '211630', 'n_tokens_mistral': 1249, 'n_tokens_neox': 1142, 'n_words': 451}