<commit_before>package fetchers

import (
	"code.google.com/p/cascadia"
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"strconv"
	"time"
)

type KatRow struct {
	Name   string
	Magnet string
	Size   uint64
Files  uint32
	Age    time.Time
}

func parseSize(amount float64, qty string) (uint64, error) {
	switch qty {
	case "bytes":
		return uint64(amount), nil
	case "KB":
		return uint64(amount * 1024), nil
	case "MB":
		return uint64(amount * 1024 * 1024), nil
	case "GB":
		return uint64(amount * 1024 * 1024 * 1024), nil
	default:
		return 0, errors.New(fmt.Sprintf("quantity inside html is not recognized: %v\n", qty))
	}
}

func NewKatRow(n *html.Node) (k *KatRow) {
	nameSelector := cascadia.MustCompile(".cellMainLink")
	name := nameSelector.MatchFirst(n).FirstChild.Data

	magnetSelector := cascadia.MustCompile(".imagnet")
	var magnet string
	for _, attribute := range magnetSelector.MatchFirst(n).Attr {
		if attribute.Key == "href" {
			magnet = attribute.Val
			break
		}
	}

	sizeSelector := cascadia.MustCompile(".nobr.center")
	sizeEl := sizeSelector.MatchFirst(n).FirstChild
	sizeQty := sizeEl.NextSibling.FirstChild.Data

	sizeAmount , err := strconv.ParseFloat(sizeEl.Data, 64)
	if err != nil {
		fmt.Printf("trying to parse: %#v", sizeEl.Data)
		return
	}
	size, err := parseSize(sizeAmount, sizeQty)
	if err != nil {
		fmt.Println("Error sizesss!!!")
		return nil
	}

	return &KatRow{Name: name, Magnet: magnet, Size: size}
}<commit_msg>exploring possibilities of using smaller data sizes<commit_after>package fetchers

import (
	"code.google.com/p/cascadia"
	"code.google.com/p/go.net/html"
	"errors"
	"fmt"
	"strconv"
	"time"
)

type KatRow struct {
	Name   string
	Magnet string
	Size   uint64
Files  uint32
	Age    time.Time
}

func parseSize(amount float32, qty string) (uint64, error) {
	switch qty {
	case "bytes":
		return uint64(amount), nil
	case "KB":
		return uint64(amount * 1024), nil
	case "MB":
		return uint64(amount * 1024 * 1024), nil
	case "GB":
		return uint64(amount * 1024 * 1024 * 1024), nil
	default:
		return 0, errors.New(fmt.Sprintf("quantity inside html is not recognized: %v\n", qty))
	}
}

func NewKatRow(n *html.Node) (k *KatRow) {
	nameSelector := cascadia.MustCompile(".cellMainLink")
	name := nameSelector.MatchFirst(n).FirstChild.Data

	magnetSelector := cascadia.MustCompile(".imagnet")
	var magnet string
	for _, attribute := range magnetSelector.MatchFirst(n).Attr {
		if attribute.Key == "href" {
			magnet = attribute.Val
			break
		}
	}

	sizeSelector := cascadia.MustCompile(".nobr.center")
	sizeEl := sizeSelector.MatchFirst(n).FirstChild
	sizeQty := sizeEl.NextSibling.FirstChild.Data

	sizeAmount , err := strconv.ParseFloat(sizeEl.Data, 32)
	if err != nil {
		fmt.Printf("trying to parse: %#v", sizeEl.Data)
		return
	}
	size, err := parseSize(float32(sizeAmount), sizeQty)
	if err != nil {
		fmt.Println("Error sizesss!!!")
		return nil
	}

	return &KatRow{Name: name, Magnet: magnet, Size: size}
}
{'fraction_non_alphanumeric': 0.13665432514304948, 'fraction_numerical': 0.029619656681252104, 'mean_word_length': 4.18673647469459, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7437591', 'n_tokens_mistral': 1204, 'n_tokens_neox': 1075, 'n_words': 264}