<commit_before>// Copyright ©2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	// job is the scripting interface to 'fwk'
	"go-hep.org/x/hep/fwk/job"

	// side-effect import 'testdata'.
	// merely importing it will register the components defined in this package
	// with the fwk components' factory.
	_ "go-hep.org/x/hep/fwk/internal/fwktest"
)

var (
	lvl    = flag.String("l", "INFO", "message level (DEBUG|INFO|WARN|ERROR)")
	evtmax = flag.Int64("evtmax", 10, "number of events to process")
	nprocs = flag.Int("nprocs", -1, "number of events to process concurrently")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %[1]s [options]

ex:
 $ %[1]s -l=INFO -evtmax=-1

options:
`,
			os.Args[0],
		)
		flag.PrintDefaults()
	}

	flag.Parse()<commit_msg>fwk: manual fix of the last reference of pkg testdata<commit_after>// Copyright ©2017 The go-hep Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	// job is the scripting interface to 'fwk'
	"go-hep.org/x/hep/fwk/job"

	// side-effect import 'fwktest'.
	// merely importing it will register the components defined in this package
	// with the fwk components' factory.
	_ "go-hep.org/x/hep/fwk/internal/fwktest"
)

var (
	lvl    = flag.String("l", "INFO", "message level (DEBUG|INFO|WARN|ERROR)")
	evtmax = flag.Int64("evtmax", 10, "number of events to process")
	nprocs = flag.Int("nprocs", -1, "number of events to process concurrently")
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %[1]s [options]

ex:
 $ %[1]s -l=INFO -evtmax=-1

options:
`,
			os.Args[0],
		)
		flag.PrintDefaults()
	}

	flag.Parse()
{'fraction_non_alphanumeric': 0.14960629921259844, 'fraction_numerical': 0.013648293963254593, 'mean_word_length': 3.453271028037383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5305335', 'n_tokens_mistral': 745, 'n_tokens_neox': 701, 'n_words': 214}