<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Import docs from a git branch and format them for gh-pages.
// usage: go run _tools/release_docs/main.go _tools/release_docs/api-reference-process.go --branch release-1.0 --output-dir v1.0
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	branch       = flag.String("branch", "", "The git branch from which to pull docs. (e.g. release-1.0, master).")
	outputDir    = flag.String("output-dir", "", "The directory in which to save results.")
	remote       = flag.String("remote", "upstream", "The name of the remote repo from which to pull docs.")
	apiReference = flag.Bool("apiReference", true, "Whether update api reference")

	subdirs = []string{"docs", "examples"}
)

func fileExistsInBranch(path string) bool {
	out, err := exec.Command("git", "ls-tree", fmt.Sprintf("%s/%s", *remote, *branch), path).Output()
	return err == nil && len(out) != 0
}

func fixURL(filename string, u *url.URL) bool {
	if u.Host != "" || u.Path == "" {
		return false
	}

	target := filepath.Join(filepath.Dir(filename), u.Path)
	if fi, err := os.Stat(target); os.IsNotExist(err) {
		// We're linking to something we didn't copy over. Send
		// it through the redirector.
		rel, err := filepath.Rel(*outputDir, target)
		if err != nil {
			return false
		}

		if fileExistsInBranch(rel) {
			u.Path = filepath.Join("HEAD", rel)
			u.Host = "releases.k8s.io"
			u.Scheme = "https"
			return true
		}
	} else if fi.IsDir() {
		// If there's no README.md in the directory, redirect to github
		// for the directory view.
		files, err := filepath.Glob(target + "/*")
		if err != nil {
			return false
		}

		hasReadme := false
		for _, f := range files {
			fmt.Println("KRIS:", f)
			if strings.ToLower(filepath.Base(f)) == "readme.md" {
				hasReadme = true
			}
		}<commit_msg>Update main.go<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Import docs from a git branch and format them for gh-pages.
// usage: go run _tools/release_docs/main.go _tools/release_docs/api-reference-process.go --branch release-1.0 --output-dir v1.0
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	branch       = flag.String("branch", "", "The git branch from which to pull docs. (e.g. release-1.0, master).")
	outputDir    = flag.String("output-dir", "", "The directory in which to save results.")
	remote       = flag.String("remote", "upstream", "The name of the remote repo from which to pull docs.")
	apiReference = flag.Bool("apiReference", true, "Whether update api reference")

	subdirs = []string{"docs", "examples"}
)

func fileExistsInBranch(path string) bool {
	out, err := exec.Command("git", "ls-tree", fmt.Sprintf("%s/%s", *remote, *branch), path).Output()
	return err == nil && len(out) != 0
}

func fixURL(filename string, u *url.URL) bool {
	if u.Host != "" || u.Path == "" {
		return false
	}

	target := filepath.Join(filepath.Dir(filename), u.Path)
	if fi, err := os.Stat(target); os.IsNotExist(err) {
		// We're linking to something we didn't copy over. Send
		// it through the redirector.
		rel, err := filepath.Rel(*outputDir, target)
		if err != nil {
			return false
		}

		if fileExistsInBranch(rel) {
			u.Path = filepath.Join("HEAD", rel)
			u.Host = "releases.k8s.io"
			u.Scheme = "https"
			return true
		}
	} else if fi.IsDir() {
		// If there's no README.md in the directory, redirect to github
		// for the directory view.
		files, err := filepath.Glob(target + "/*")
		if err != nil {
			return false
		}

		hasReadme := false
		for _, f := range files {
			if strings.ToLower(filepath.Base(f)) == "readme.md" {
				hasReadme = true
			}
		}
{'fraction_non_alphanumeric': 0.13528932355338225, 'fraction_numerical': 0.006519967400162999, 'mean_word_length': 3.4995417048579287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '785724', 'n_tokens_mistral': 1725, 'n_tokens_neox': 1596, 'n_words': 551}