<commit_before>// Copyright © 2013-14 <NAME> <<EMAIL>>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/spf13/hugo/hugofs"
	jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/viper"
)

const pygmentsBin = "pygmentize"

// HasPygments checks to see if Pygments is installed and available
// on the system.
func HasPygments() bool {
	if _, err := exec.LookPath(pygmentsBin); err != nil {
		return false
	}
	return true
}

// Highlight takes some code and returns highlighted code.
func Highlight(code string, lexer string) string {

	if !HasPygments() {
		jww.WARN.Println("Highlighting requires Pygments to be installed and in the path")
		return code
	}

	fs := hugofs.OsFs

	// Try to read from cache first
	hash := sha1.Sum([]byte(code))
	cachefile := fmt.Sprintf("%s/pygments-%s-%x", viper.GetString("CacheDir"), lexer, hash)
	exists, err := Exists(cachefile, fs)
	if err != nil {
		jww.ERROR.Print(err.Error())
		return code
	}
	if exists {
		f, err := fs.Open(cachefile)
		if err != nil {
			jww.ERROR.Print(err.Error())
			return code
		}

		s, err := ioutil.ReadAll(f)
		if err != nil {
			jww.ERROR.Print(err.Error())
			return code
		}

		return string(s)
	}

	// No cache file, render and cache it
	var out bytes.Buffer
	var stderr bytes.Buffer
	style := viper.GetString("PygmentsStyle")

	noclasses := "true"
	if viper.GetBool("PygmentsUseClasses") {
		noclasses = "false"
	}

	cmd := exec.Command(pygmentsBin, "-l"+lexer, "-fhtml", "-O",
		fmt.Sprintf("style=%s,noclasses=%s,encoding=utf8", style, noclasses))
	cmd.Stdin = strings.NewReader(code)
	cmd.Stdout = &out
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		jww.ERROR.Print(stderr.String())
		return code
	}

	// Write cache file
	if err := WriteToDisk(cachefile, bytes.NewReader(out.Bytes()), fs); err != nil {
		jww.ERROR.Print(stderr.String())
	}

	return out.String()<commit_msg>Hash all pygments parameters.<commit_after>// Copyright © 2013-14 <NAME> <<EMAIL>>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"bytes"
	"crypto/sha1"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/spf13/hugo/hugofs"
	jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/viper"
)

const pygmentsBin = "pygmentize"

// HasPygments checks to see if Pygments is installed and available
// on the system.
func HasPygments() bool {
	if _, err := exec.LookPath(pygmentsBin); err != nil {
		return false
	}
	return true
}

// Highlight takes some code and returns highlighted code.
func Highlight(code string, lexer string) string {

	if !HasPygments() {
		jww.WARN.Println("Highlighting requires Pygments to be installed and in the path")
		return code
	}

	fs := hugofs.OsFs

	style := viper.GetString("PygmentsStyle")

	noclasses := "true"
	if viper.GetBool("PygmentsUseClasses") {
		noclasses = "false"
	}

	// Try to read from cache first
	hash := sha1.New()
	io.WriteString(hash, lexer)
	io.WriteString(hash, code)
	io.WriteString(hash, style)
	io.WriteString(hash, noclasses)

	cachefile := fmt.Sprintf("%s/pygments-%x", viper.GetString("CacheDir"), hash.Sum(nil))
	exists, err := Exists(cachefile, fs)
	if err != nil {
		jww.ERROR.Print(err.Error())
		return code
	}
	if exists {
		f, err := fs.Open(cachefile)
		if err != nil {
			jww.ERROR.Print(err.Error())
			return code
		}

		s, err := ioutil.ReadAll(f)
		if err != nil {
			jww.ERROR.Print(err.Error())
			return code
		}

		return string(s)
	}

	// No cache file, render and cache it
	var out bytes.Buffer
	var stderr bytes.Buffer

	cmd := exec.Command(pygmentsBin, "-l"+lexer, "-fhtml", "-O",
		fmt.Sprintf("style=%s,noclasses=%s,encoding=utf8", style, noclasses))
	cmd.Stdin = strings.NewReader(code)
	cmd.Stdout = &out
	cmd.Stderr = &stderr

	if err := cmd.Run(); err != nil {
		jww.ERROR.Print(stderr.String())
		return code
	}

	// Write cache file
	if err := WriteToDisk(cachefile, bytes.NewReader(out.Bytes()), fs); err != nil {
		jww.ERROR.Print(stderr.String())
	}

	return out.String()
{'fraction_non_alphanumeric': 0.13572267920094008, 'fraction_numerical': 0.007442224833529181, 'mean_word_length': 3.7025782688766116, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6496292', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1742, 'n_words': 524}