<commit_before>// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

// gen.go converts "../base/*" to "data.go".
//
// Invoke it via "go generate".

import (
	"bytes"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"strings"
)

const columns = 1024

func main() {
	if err := main1(); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func main1() error {
	out := &bytes.Buffer{}
	out.WriteString("// Code generated by running \"go generate\". DO NOT EDIT.\n")
	out.WriteString("\n")
	out.WriteString("// Copyright 2017 The Wuffs Authors.\n")
	out.WriteString("//\n")
	out.WriteString("// Licensed under the Apache License, Version 2.0 (the \"License\");\n")
	out.WriteString("// you may not use this file except in compliance with the License.\n")
	out.WriteString("// You may obtain a copy of the License at\n")
	out.WriteString("//\n")
	out.WriteString("//    https://www.apache.org/licenses/LICENSE-2.0\n")
	out.WriteString("//\n")
	out.WriteString("// Unless required by applicable law or agreed to in writing, software\n")
	out.WriteString("// distributed under the License is distributed on an \"AS IS\" BASIS,\n")
	out.WriteString("// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n")
	out.WriteString("// See the License for the specific language governing permissions and\n")
	out.WriteString("// limitations under the License.\n")
	out.WriteString("\n")
	out.WriteString("package data\n")
	out.WriteString("\n")

	if err := genData(out); err != nil {
		return err
	}

	formatted, err := format.Source(out.Bytes())
	if err != nil {
		return err
	}
	return ioutil.WriteFile("data.go", formatted, 0644)
}

func genData(out *bytes.Buffer) error {
	files := []struct {
		filename, varname string
	}{
		{"../base/all-impl.c", "BaseAllImplC"},

		{"../base/fundamental-private.h", "BaseFundamentalPrivateH"},
		{"../base/fundamental-public.h", "BaseFundamentalPublicH"},
		{"../base/memory-private.h", "BaseMemoryPrivateH"},
		{"../base/memory-public.h", "BaseMemoryPublicH"},
		{"../base/image-private.h", "BaseImagePrivateH"},
		{"../base/image-public.h", "BaseImagePublicH"},
		{"../base/io-private.h", "BaseIOPrivateH"},
		{"../base/io-public.h", "BaseIOPublicH"},
		{"../base/range-private.h", "BaseRangePrivateH"},
		{"../base/range-public.h", "BaseRangePublicH"},
		{"../base/strconv-private.h", "BaseStrConvPrivateH"},
		{"../base/strconv-public.h", "BaseStrConvPublicH"},
		{"../base/token-private.h", "BaseTokenPrivateH"},
		{"../base/token-public.h", "BaseTokenPublicH"},

		{"../base/floatconv-submodule-code.c", "BaseFloatConvSubmoduleCodeC"},
		{"../base/floatconv-submodule-data.c", "BaseFloatConvSubmoduleDataC"},
		{"../base/intconv-submodule.c", "BaseIntConvSubmoduleC"},
		{"../base/pixconv-submodule.c", "BasePixConvSubmoduleC"},
		{"../base/utf8-submodule.c", "BaseUTF8SubmoduleC"},

		{"../aux/base.cc", "AuxBaseCc"},
		{"../aux/base.hh", "AuxBaseHh"},
		{"../aux/json.cc", "AuxJsonCc"},
		{"../aux/json.hh", "AuxJsonHh"},
	}

	prefixAfterEditing := []byte("// After editing this file,")
	prefixCopyright := []byte("// Copyright ")
	copyright := []byte(nil)

	for _, f := range files {
		in, err := ioutil.ReadFile(f.filename)
		if err != nil {
			return err
		}<commit_msg>updating refs to new auxiliary folder<commit_after>// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build ignore

package main

// gen.go converts "../base/*" to "data.go".
//
// Invoke it via "go generate".

import (
	"bytes"
	"fmt"
	"go/format"
	"io/ioutil"
	"os"
	"strings"
)

const columns = 1024

func main() {
	if err := main1(); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func main1() error {
	out := &bytes.Buffer{}
	out.WriteString("// Code generated by running \"go generate\". DO NOT EDIT.\n")
	out.WriteString("\n")
	out.WriteString("// Copyright 2017 The Wuffs Authors.\n")
	out.WriteString("//\n")
	out.WriteString("// Licensed under the Apache License, Version 2.0 (the \"License\");\n")
	out.WriteString("// you may not use this file except in compliance with the License.\n")
	out.WriteString("// You may obtain a copy of the License at\n")
	out.WriteString("//\n")
	out.WriteString("//    https://www.apache.org/licenses/LICENSE-2.0\n")
	out.WriteString("//\n")
	out.WriteString("// Unless required by applicable law or agreed to in writing, software\n")
	out.WriteString("// distributed under the License is distributed on an \"AS IS\" BASIS,\n")
	out.WriteString("// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n")
	out.WriteString("// See the License for the specific language governing permissions and\n")
	out.WriteString("// limitations under the License.\n")
	out.WriteString("\n")
	out.WriteString("package data\n")
	out.WriteString("\n")

	if err := genData(out); err != nil {
		return err
	}

	formatted, err := format.Source(out.Bytes())
	if err != nil {
		return err
	}
	return ioutil.WriteFile("data.go", formatted, 0644)
}

func genData(out *bytes.Buffer) error {
	files := []struct {
		filename, varname string
	}{
		{"../base/all-impl.c", "BaseAllImplC"},

		{"../base/fundamental-private.h", "BaseFundamentalPrivateH"},
		{"../base/fundamental-public.h", "BaseFundamentalPublicH"},
		{"../base/memory-private.h", "BaseMemoryPrivateH"},
		{"../base/memory-public.h", "BaseMemoryPublicH"},
		{"../base/image-private.h", "BaseImagePrivateH"},
		{"../base/image-public.h", "BaseImagePublicH"},
		{"../base/io-private.h", "BaseIOPrivateH"},
		{"../base/io-public.h", "BaseIOPublicH"},
		{"../base/range-private.h", "BaseRangePrivateH"},
		{"../base/range-public.h", "BaseRangePublicH"},
		{"../base/strconv-private.h", "BaseStrConvPrivateH"},
		{"../base/strconv-public.h", "BaseStrConvPublicH"},
		{"../base/token-private.h", "BaseTokenPrivateH"},
		{"../base/token-public.h", "BaseTokenPublicH"},

		{"../base/floatconv-submodule-code.c", "BaseFloatConvSubmoduleCodeC"},
		{"../base/floatconv-submodule-data.c", "BaseFloatConvSubmoduleDataC"},
		{"../base/intconv-submodule.c", "BaseIntConvSubmoduleC"},
		{"../base/pixconv-submodule.c", "BasePixConvSubmoduleC"},
		{"../base/utf8-submodule.c", "BaseUTF8SubmoduleC"},

		{"../auxiliary/base.cc", "AuxBaseCc"},
		{"../auxiliary/base.hh", "AuxBaseHh"},
		{"../auxiliary/json.cc", "AuxJsonCc"},
		{"../auxiliary/json.hh", "AuxJsonHh"},
	}

	prefixAfterEditing := []byte("// After editing this file,")
	prefixCopyright := []byte("// Copyright ")
	copyright := []byte(nil)

	for _, f := range files {
		in, err := ioutil.ReadFile(f.filename)
		if err != nil {
			return err
		}
{'fraction_non_alphanumeric': 0.1881342701014832, 'fraction_numerical': 0.0075461878740567265, 'mean_word_length': 3.704406364749082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276388', 'n_tokens_mistral': 2669, 'n_tokens_neox': 2532, 'n_words': 671}