<commit_before>package jq

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/jingweno/jqpipe-go"
)

var Path, Version string

func init() {
	var err error

	Path, err = setupJQPath()
	if err != nil {
		log.Fatal(err)
	}

	Version, err = jqVersion()
	if err != nil {
		log.Fatal(err)
	}
}

func setupJQPath() (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	jqPath := filepath.Join(pwd, "bin", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
	os.Setenv("PATH", fmt.Sprintf("%s%c%s", jqPath, os.PathListSeparator, os.Getenv("PATH")))

	return filepath.Join(jqPath, "jq"), nil
}

func jqVersion() (string, error) {
	out, err := exec.Command("jq", "--version").Output()
	if err != nil {
		return "", err
	}

	out = bytes.TrimSpace(out)
	r := regexp.MustCompile(`^jq-(.+)$`)
	if r.Match(out) {
		m := r.FindSubmatch(out)[1]
		return string(m), nil
	}

	return "", fmt.Errorf("can't find jq version")
}

type JQ struct {
	J string `json:"j"`
	Q string `json:"q"`
}

func (j *JQ) Eval() (string, error) {
	seq, err := jq.Eval(j.J, j.Q)
	if err != nil {
		return "", err
	}

	result := []string{}
	for _, s := range seq {
		ss := string(s)
		if ss != "" && ss != "null" {
			result = append(result, ss)
		}
	}

	return strings.Join(result, "\n"), nil
}

func (j *JQ) Valid() bool {
	return j.J != "" && j.Q != ""
}<commit_msg>Fallback solution to get jq version<commit_after>package jq

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/jingweno/jqpipe-go"
)

var Path, Version string

func init() {
	var err error

	Path, err = setupJQPath()
	if err != nil {
		log.Fatal(err)
	}

	Version, err = jqVersion()
	if err != nil {
		log.Fatal(err)
	}
}

func setupJQPath() (string, error) {
	pwd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	jqPath := filepath.Join(pwd, "bin", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
	os.Setenv("PATH", fmt.Sprintf("%s%c%s", jqPath, os.PathListSeparator, os.Getenv("PATH")))

	return filepath.Join(jqPath, "jq"), nil
}

func jqVersion() (string, error) {
	out, err := exec.Command("jq", "--version").Output()
	if err != nil {
		return "", err
	}

	out = bytes.TrimSpace(out)
	r := regexp.MustCompile(`^jq-(.+)$`)
	if r.Match(out) {
		m := r.FindSubmatch(out)[1]
		return string(m), nil
	}

	r = regexp.MustCompile(`^jq version (.+)$`)
	if r.Match(out) {
		m := r.FindSubmatch(out)[1]
		return string(m), nil
	}

	return "", fmt.Errorf("can't find jq version")
}

type JQ struct {
	J string `json:"j"`
	Q string `json:"q"`
}

func (j *JQ) Eval() (string, error) {
	seq, err := jq.Eval(j.J, j.Q)
	if err != nil {
		return "", err
	}

	result := []string{}
	for _, s := range seq {
		ss := string(s)
		if ss != "" && ss != "null" {
			result = append(result, ss)
		}
	}

	return strings.Join(result, "\n"), nil
}

func (j *JQ) Valid() bool {
	return j.J != "" && j.Q != ""
}
{'fraction_non_alphanumeric': 0.207997311827957, 'fraction_numerical': 0.0010080645161290322, 'mean_word_length': 3.4103703703703703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3325036', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1242, 'n_words': 301}