<commit_before>package main

import (
	"bufio"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

type clippingHandler func(*rawClipping)

type parser struct {
	handler clippingHandler
}

func (p *parser) parseClippingFile(clippingFile string) {
	f, err := os.Open(clippingFile)
	if err != nil {
		log.Fatalln("Cannot open file", err.Error())
	}
	defer f.Close()

	p.parse(f)
}

type clippingType int

const (
	undefined clippingType = iota
	highlight
	note
	bookmark
)

type rawClipping struct {
	baseClipping
	cType clippingType
}

func (i *parser) parse(r io.Reader) {
	c := new(rawClipping)

	lineNo := 0

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		lineNo++

		if line == "==========" {
			lineNo = 0
			if c != nil {
				i.handler(c)
			}
			c = new(rawClipping)
			continue
		}

		switch lineNo {
		case 1:
			i.extractBook(line, c)
		case 2:
			i.extractLocationAndDate(line, c)
		case 3:
			continue
		default:
			if c.Content != "" {
				c.Content += "\n"
			}
			c.Content += line
		}

	}
}

func (i *parser) extractBook(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " (")
	if ix < 0 {
		c.Book.Title = str
	} else {
		c.Book.Title = str[:ix]<commit_msg>Remove BOM from book titles<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

type clippingHandler func(*rawClipping)

type parser struct {
	handler clippingHandler
}

var bom = []byte{0xef, 0xbb, 0xbf}

func (p *parser) parseClippingFile(clippingFile string) {
	f, err := os.Open(clippingFile)
	if err != nil {
		log.Fatalln("Cannot open file", err.Error())
	}
	defer f.Close()

	p.parse(f)
}

type clippingType int

const (
	undefined clippingType = iota
	highlight
	note
	bookmark
)

type rawClipping struct {
	baseClipping
	cType clippingType
}

func (i *parser) parse(r io.Reader) {
	c := new(rawClipping)

	lineNo := 0

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		lineNo++

		if line == "==========" {
			lineNo = 0
			if c != nil {
				i.handler(c)
			}
			c = new(rawClipping)
			continue
		}

		switch lineNo {
		case 1:
			i.extractBook(line, c)
		case 2:
			i.extractLocationAndDate(line, c)
		case 3:
			continue
		default:
			if c.Content != "" {
				c.Content += "\n"
			}
			c.Content += line
		}

	}
}

func (i *parser) extractBook(str string, c *rawClipping) {
	if bytes.HasPrefix([]byte(str), bom) {
		str = str[3:]
	}
	ix := strings.LastIndex(str, " (")
	if ix < 0 {
		c.Book.Title = str
	} else {
		c.Book.Title = str[:ix]
{'fraction_non_alphanumeric': 0.14929467084639497, 'fraction_numerical': 0.006269592476489028, 'mean_word_length': 3.1377633711507293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693674', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1017, 'n_words': 246}