<commit_before>package dicom

import (
	"encoding/csv"
	"io"
	"strconv"
	"strings"
)

type dictEntry struct {
	vr      string
	name    string
	vm      string
	version string
}

func (p *Parser) loadDictionary(dictionary io.Reader) error {
	reader := csv.NewReader(dictionary)
	reader.Comma = '\t'  // tab separated file
	reader.Comment = '#' // comments start with #

	p.dict = make([][]*dictEntry, 0xffff+1)

	for {
		row, err := reader.Read()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		tag := strings.Split(strings.Trim(row[0], "()"), ",")

		group, err := strconv.ParseInt(tag[0], 16, 0)
		if err != nil {
			// TODO: Get this to work for ranges
			// return err
			continue
		}
		element, err := strconv.ParseInt(tag[1], 16, 0)
		if err != nil {
			// return err
			continue
		}

		if cap(p.dict[group]) == 0 {

			p.dict[group] = make([]*dictEntry, 0xffff+1)
		}

		p.dict[group][element] = &dictEntry{row[1], row[2], row[3], row[4]}
	}
	return nil
}

func (p *Parser) getDictEntry(group, element int) (*dictEntry, error) {
	entry := p.dict[group][element]
	if entry == nil {
		return nil, ErrTagNotFound
	}

	return entry, nil
}<commit_msg>Refactored into separate splitTags function<commit_after>package dicom

import (
	"encoding/csv"
	"io"
	"strconv"
	"strings"
)

type dictEntry struct {
	vr      string
	name    string
	vm      string
	version string
}

func (p *Parser) loadDictionary(dictionary io.Reader) error {
	reader := csv.NewReader(dictionary)
	reader.Comma = '\t'  // tab separated file
	reader.Comment = '#' // comments start with #

	p.dict = make([][]*dictEntry, 0xffff+1)

	for {
		row, err := reader.Read()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		group, element, err := splitTag(row[0])
		if err != nil {
			// TODO: fix group ranges (6000-60FF,0803) causing error
			continue
		}

		if cap(p.dict[group]) == 0 {

			p.dict[group] = make([]*dictEntry, 0xffff+1)
		}

		p.dict[group][element] = &dictEntry{row[1], row[2], row[3], row[4]}
	}
	return nil
}

func (p *Parser) getDictEntry(group, element int) (*dictEntry, error) {
	entry := p.dict[group][element]
	if entry == nil {
		return nil, ErrTagNotFound
	}

	return entry, nil
}

// Split a tag into a group and element, represented as a hex value
// TODO: support group ranges (6000-60FF,0803)
func splitTag(tag string) (int64, int64, error) {

	parts := strings.Split(strings.Trim(tag, "()"), ",")

	group, err := strconv.ParseInt(parts[0], 16, 0)
	if err != nil {
		return 0, 0, err
	}
	elem, err := strconv.ParseInt(parts[1], 16, 0)
	if err != nil {
		return 0, 0, err
	}

	return group, elem, nil
}
{'fraction_non_alphanumeric': 0.1562734785875282, 'fraction_numerical': 0.02404207362885049, 'mean_word_length': 3.2472089314194577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2792931', 'n_tokens_mistral': 1086, 'n_tokens_neox': 967, 'n_words': 274}