<commit_before>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"os"
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	var delim string = ""
	for _, chunk := range chunks {
		switch {
		case last == nil:
			delim = ""
		case last.Block != chunk.Block:
			delim = "\n\n"
		case last.Block == chunk.Block:
			delim = " "
		}
		fmt.Printf("%s%s", delim, chunk.Text)
		last = chunk
	}
	fmt.Println()
}

func main() {
	ext := model.NewExtractor()
	for _, arg := range os.Args[1:] {
		file, err := os.Open(arg)
		if err != nil {
			panic(err)
		}
		defer file.Close()
<commit_msg>output bold text for headings and emphasized elements<commit_after>package main

import (
	"fmt"
	"github.com/slyrz/newscat/html"
	"github.com/slyrz/newscat/model"
	"os"
)

func printChunks(chunks []*html.Chunk) {
	var last *html.Chunk = nil
	var delim string = ""
	var pre string = ""
	var pos string = ""
	for _, chunk := range chunks {
		// If the last chunk and the current chunk share the same HTML block element,
		// we seperate them by a space character. If they are in different blocks, we
		// use two newline characters.
		switch {
		case last == nil:
			delim = ""
		case last.Block != chunk.Block:
			delim = "\n\n"
		case last.Block == chunk.Block:
			delim = " "
		}
		// Use bold font for headings, emphasized and bold text.
		switch chunk.Base.Data {
			case "h1", "h2", "h3", "h4", "h5", "h6", "em", "strong", "b":
				pre, pos = "\x1b[39;1m", "\x1b[0m"
			default:
				pre, pos = "", ""
		}
		fmt.Printf("%s%s%s%s", delim, pre, chunk.Text, pos)
		last = chunk
	}
	fmt.Println()
}

func main() {
	ext := model.NewExtractor()
	for _, arg := range os.Args[1:] {
		file, err := os.Open(arg)
		if err != nil {
			panic(err)
		}
		defer file.Close()

{'fraction_non_alphanumeric': 0.16417910447761194, 'fraction_numerical': 0.007739082365948038, 'mean_word_length': 2.978021978021978, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '838301', 'n_tokens_mistral': 761, 'n_tokens_neox': 692, 'n_words': 191}