<commit_before>package main

import (
	"strings"

	"github.com/animenotifier/arn"
)

func parseCharacterDescription(input string) (output string, attributes []*arn.CharacterAttribute) {
	// Parse attributes like these:
	// - Position: Club Manager
	// - Height: 162 cm (5' 4")
	// - Weight: 48 kg (106 lb)
	// - Birthday: November 24
	// - Hair color: Brown
	// - Eyes: Blue (anime), Green (manga)

	paragraphs := strings.Split(input, "\n\n")
	finalParagraphs := make([]string, 0, len(paragraphs))

	for _, paragraph := range paragraphs {
		// Is paragraph full of attributes?
		if strings.Contains(paragraph, "\n") {
			lines := strings.Split(paragraph, "\n")
			var lastAttribute *arn.CharacterAttribute

			for _, line := range lines {
				line = strings.Replace(line, " (\n)", "", -1)

				// Remove all kinds of starting and ending parantheses.
				if strings.HasPrefix(line, "(") {
					line = strings.TrimPrefix(line, "(")
					line = strings.TrimSuffix(line, ")")
				}

				line = strings.TrimSuffix(line, " (")
				line = strings.TrimPrefix(line, ")")

				parts := strings.Split(line, ":")

				if len(parts) < 2 {
					// Add to previous attribute
					if lastAttribute != nil {
						lastAttribute.Value += ", " + line
					}

					continue
				}

				name := strings.TrimSpace(parts[0])
				value := strings.TrimSpace(parts[1])

				if value == "" || value == `"` {
					continue
				}<commit_msg>Updated MAL sync<commit_after>package main

import (
	"strings"

	"github.com/animenotifier/arn"
)

func parseCharacterDescription(input string) (output string, attributes []*arn.CharacterAttribute) {
	// Parse attributes like these:
	// - Position: Club Manager
	// - Height: 162 cm (5' 4")
	// - Weight: 48 kg (106 lb)
	// - Birthday: November 24
	// - Hair color: Brown
	// - Eyes: Blue (anime), Green (manga)

	paragraphs := strings.Split(input, "\n\n")
	finalParagraphs := make([]string, 0, len(paragraphs))

	for _, paragraph := range paragraphs {
		// Is paragraph full of attributes?
		if strings.Contains(paragraph, "\n") {
			lines := strings.Split(paragraph, "\n")
			var lastAttribute *arn.CharacterAttribute

			for _, line := range lines {
				// line = strings.Replace(line, " (\n)", "", -1)

				// Remove all kinds of starting and ending parantheses.
				if strings.HasPrefix(line, "(") {
					line = strings.TrimPrefix(line, "(")
					line = strings.TrimSuffix(line, ")")
				}

				if !strings.Contains(line, ":") {
					// Remove list indicators
					line = strings.TrimPrefix(line, "- ")
					line = strings.TrimPrefix(line, "* ")

					// Add to previous attribute
					if lastAttribute != nil {
						if lastAttribute.Value != "" {
							lastAttribute.Value += ", "
						}

						lastAttribute.Value += line
					}

					continue
				}

				parts := strings.Split(line, ":")
				name := strings.TrimSpace(parts[0])
				value := strings.TrimSpace(parts[1])
{'fraction_non_alphanumeric': 0.15600138360428917, 'fraction_numerical': 0.011414735385679695, 'mean_word_length': 2.8870967741935485, 'pattern_counts': {'":': 3, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5494200', 'n_tokens_mistral': 1120, 'n_tokens_neox': 881, 'n_words': 231}