<commit_before>package util

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"strings"

	"github.com/go-aah/website/app/markdown"
	"github.com/go-aah/website/app/models"

	"aahframework.org/aah.v0"
	"aahframework.org/ahttp.v0"
	"aahframework.org/essentials.v0"
	"aahframework.org/log.v0"
)

var (
	releases      []string
	editURLPrefix string
)

// BranchName method returns the confirmed branch name
func BranchName(version string) string {
	releases, _ := aah.AppConfig().StringList("docs.releases")
	if version == releases[0] {
		return "master"
	}
	return version
}

// DocBaseDir method returns the aah documentation based directory.
func DocBaseDir() string {
	return filepath.Join(aah.AppConfig().StringDefault("docs.dir", ""), "aah-documentation")
}

// DocVersionBaseDir method returns the documentation dir path for given
// language and version.
func DocVersionBaseDir(version string) string {
	return filepath.Join(DocBaseDir(), version)
}

// RefreshDocContent method clone's the GitHub branch doc version wise into
// local and if already exits it takes a update from GitHub.
// It clears cache too.
func RefreshDocContent(pushEvent *models.GithubPushEvent) {
	version := pushEvent.BranchName()
	if version == "master" {
		version = releases[0]
	}

	if !ess.IsSliceContainsString(releases, version) {
		log.Warnf("Branch Name [%s] not found", version)
		return
	}

	GitRefresh(releases)

	log.Infof("BranchName: %s", version)
	docVersionBaseDir := DocVersionBaseDir(version)
	for _, commit := range pushEvent.Commits {
		log.Infof("CommitID: %s, Message: %s", commit.ID, commit.Message)
		log.Infof("Modified: %s, Removed: %s", commit.Modified, commit.Removed)

		for _, f := range commit.Modified {
			if strings.HasSuffix(f, "LICENSE") || strings.HasSuffix(f, "README.md") {
				continue
			}
			mdPath := FilePath(f, docVersionBaseDir)
			markdown.RefreshCacheByFile(mdPath)
		}

		for _, f := range commit.Removed {
			mdPath := FilePath(f, docVersionBaseDir)
			markdown.RemoveCacheByFile(mdPath)
		}
	}
}

// GitRefresh method clone's the GitHub branch doc version wise into
// local and if already exits it takes a update from GitHub.
func GitRefresh(releases []string) {
	for _, version := range releases {
		docDirPath := DocVersionBaseDir(version)
		branchName := BranchName(version)
		err := GitCloneAndPull(docDirPath, branchName)
		if err != nil {
			log.Error(err)
		}
	}
}

// ContentBasePath method returns the Markdown files base path.
func ContentBasePath() string {
	return filepath.Join(aah.AppVirtualBaseDir(), "content")
}

// FilePath method returns markdown file path from given path.
// it bacially remove any extension and adds ".md"
func FilePath(reqPath, prefix string) string {
	reqPath = strings.ToLower(TrimPrefixSlash(reqPath))
	reqPath = ess.StripExt(reqPath) + ".md"
	return filepath.Clean(filepath.Join(prefix, reqPath))
}

// TrimPrefixSlash method trims the prefix slash from the given path
func TrimPrefixSlash(str string) string {
	return strings.TrimPrefix(str, "/")
}

// CreateKey method creates markdown file name from request path.
func CreateKey(rpath string) string {
	key := ess.StripExt(TrimPrefixSlash(rpath))
	return strings.Replace(key, "/", "-", -1)
}

// PullGithubDocsAndLoadCache method pulls github docs and populate documentation
// in the cache
func PullGithubDocsAndLoadCache(e *aah.Event) {
	cfg := aah.AppConfig()
	editURLPrefix = cfg.StringDefault("docs.edit_url_prefix", "")
	releases, _ = cfg.StringList("docs.releases")

	keepLocalVersion := os.Getenv("KEEP_FILES")
	if len(keepLocalVersion) > 0 {
		return
	}

	docBasePath := DocBaseDir()

	if aah.AppProfile() == "prod" {
		ess.DeleteFiles(docBasePath)
	}<commit_msg>now vfs provides fs indepentent so use path.Join<commit_after>package util

import (
	"fmt"
	"html/template"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/go-aah/website/app/markdown"
	"github.com/go-aah/website/app/models"

	"aahframework.org/aah.v0"
	"aahframework.org/ahttp.v0"
	"aahframework.org/essentials.v0"
	"aahframework.org/log.v0"
)

var (
	releases      []string
	editURLPrefix string
)

// BranchName method returns the confirmed branch name
func BranchName(version string) string {
	releases, _ := aah.AppConfig().StringList("docs.releases")
	if version == releases[0] {
		return "master"
	}
	return version
}

// DocBaseDir method returns the aah documentation based directory.
func DocBaseDir() string {
	return filepath.Join(aah.AppConfig().StringDefault("docs.dir", ""), "aah-documentation")
}

// DocVersionBaseDir method returns the documentation dir path for given
// language and version.
func DocVersionBaseDir(version string) string {
	return filepath.Join(DocBaseDir(), version)
}

// RefreshDocContent method clone's the GitHub branch doc version wise into
// local and if already exits it takes a update from GitHub.
// It clears cache too.
func RefreshDocContent(pushEvent *models.GithubPushEvent) {
	version := pushEvent.BranchName()
	if version == "master" {
		version = releases[0]
	}

	if !ess.IsSliceContainsString(releases, version) {
		log.Warnf("Branch Name [%s] not found", version)
		return
	}

	GitRefresh(releases)

	log.Infof("BranchName: %s", version)
	docVersionBaseDir := DocVersionBaseDir(version)
	for _, commit := range pushEvent.Commits {
		log.Infof("CommitID: %s, Message: %s", commit.ID, commit.Message)
		log.Infof("Modified: %s, Removed: %s", commit.Modified, commit.Removed)

		for _, f := range commit.Modified {
			if strings.HasSuffix(f, "LICENSE") || strings.HasSuffix(f, "README.md") {
				continue
			}
			mdPath := FilePath(f, docVersionBaseDir)
			markdown.RefreshCacheByFile(mdPath)
		}

		for _, f := range commit.Removed {
			mdPath := FilePath(f, docVersionBaseDir)
			markdown.RemoveCacheByFile(mdPath)
		}
	}
}

// GitRefresh method clone's the GitHub branch doc version wise into
// local and if already exits it takes a update from GitHub.
func GitRefresh(releases []string) {
	for _, version := range releases {
		docDirPath := DocVersionBaseDir(version)
		branchName := BranchName(version)
		err := GitCloneAndPull(docDirPath, branchName)
		if err != nil {
			log.Error(err)
		}
	}
}

// ContentBasePath method returns the Markdown files base path.
func ContentBasePath() string {
	return filepath.Join(aah.AppVirtualBaseDir(), "content")
}

// FilePath method returns markdown file path from given path.
// it bacially remove any extension and adds ".md"
func FilePath(reqPath, prefix string) string {
	reqPath = strings.ToLower(TrimPrefixSlash(reqPath))
	reqPath = ess.StripExt(reqPath) + ".md"
	return path.Clean(path.Join(prefix, reqPath))
}

// TrimPrefixSlash method trims the prefix slash from the given path
func TrimPrefixSlash(str string) string {
	return strings.TrimPrefix(str, "/")
}

// CreateKey method creates markdown file name from request path.
func CreateKey(rpath string) string {
	key := ess.StripExt(TrimPrefixSlash(rpath))
	return strings.Replace(key, "/", "-", -1)
}

// PullGithubDocsAndLoadCache method pulls github docs and populate documentation
// in the cache
func PullGithubDocsAndLoadCache(e *aah.Event) {
	cfg := aah.AppConfig()
	editURLPrefix = cfg.StringDefault("docs.edit_url_prefix", "")
	releases, _ = cfg.StringList("docs.releases")

	keepLocalVersion := os.Getenv("KEEP_FILES")
	if len(keepLocalVersion) > 0 {
		return
	}

	docBasePath := DocBaseDir()

	if aah.AppProfile() == "prod" {
		ess.DeleteFiles(docBasePath)
	}
{'fraction_non_alphanumeric': 0.12095583299771781, 'fraction_numerical': 0.0021479393207141898, 'mean_word_length': 4.588897224306076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6150698', 'n_tokens_mistral': 2578, 'n_tokens_neox': 2421, 'n_words': 628}