<commit_before>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package presentation

import (
	"fmt"
	"github.com/andreaskoch/allmark/parser/document"
	"github.com/andreaskoch/allmark/parser/pattern"
	"github.com/andreaskoch/allmark/repository"
	"regexp"
	"strings"
)

var (
	// markdown headline which start with the headline text righ after the hash
	markdownHeadlineStartWhitespace = regexp.MustCompile(`^(#+)([^\s#])`)
)

// Parse an item with a title, description and content
func Parse(item *repository.Item, lines []string, fallbackTitle string) (sucess bool, err error) {

	// parse the document
	if success, err := document.Parse(item, lines, fallbackTitle); !success {
		return sucess, err
	}

	// split the lines again
	presentationLines := make([]string, 0)
	lines = strings.Split(item.RawContent, "\n")

	highestHeadlineLevel := 6
	lowestHeadlineLevel := 1

	// separate the slides with horizontal rule
	for lineNumber, line := range lines {

		// skip non-headlines
		if !isHeadline(line) {
			presentationLines = append(presentationLines, line)
			continue
		}

		// determine the headline level
		headlineLevel, err := getHeadlineLevel(line)
		if err != nil {
			panic(err)
		}

		// capture the highest headline level
		if headlineLevel < highestHeadlineLevel {
			highestHeadlineLevel = headlineLevel
		}

		// capture the lowest headline level
		if headlineLevel > lowestHeadlineLevel {
			lowestHeadlineLevel = headlineLevel
		}

		// get the lines before this line
		if lineNumber > 0 && horizontalRuleAlreadyPresentIn(lines[0:lineNumber-1]) {

			// headline is already preceeded by horizontal rule
			presentationLines = append(presentationLines, line)

		} else {

			// prepend a horizontal rule
			presentationLines = append(presentationLines, "")
			presentationLines = append(presentationLines, "---")
			presentationLines = append(presentationLines, "")

			presentationLines = append(presentationLines, line)
		}

	}

	// normalize the headline levels
	// for lineNumber, line := range lines {

	// }

	// save the presentation code
	item.RawContent = strings.TrimSpace(strings.Join(presentationLines, "\n"))

	return true, nil
}

func getHeadlineLevel(line string) (int, error) {

	if !isHeadline(line) {
		return 0, fmt.Errorf("The line %q is not a headline.", line)
	}

	level := 0
	for _, character := range line {

		if string(character) == `#` {
			level++
		} else {
			break
		}
	}

	return level, nil

}

func isHeadline(line string) bool {
	return strings.HasPrefix(line, "#")
}

func horizontalRuleAlreadyPresentIn(lines []string) bool {
	if len(lines) == 0 {
		return false
	}

	for lineNumber := len(lines) - 1; lineNumber >= 0; lineNumber-- {
		line := lines[lineNumber]

		if pattern.EmptyLinePattern.MatchString(line) {
			continue
		}

		return pattern.HorizontalRulePattern.MatchString(line)
	}

	return false
}<commit_msg>Presentation Markdown Cleanup: Auto-fix headline levels. Slides with a headline but without content automatically become a h1 headline. Slides with a headline with content become a h2 headline.<commit_after>// Copyright 2013 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package presentation

import (
	"github.com/andreaskoch/allmark/parser/document"
	"github.com/andreaskoch/allmark/parser/pattern"
	"github.com/andreaskoch/allmark/repository"
	"regexp"
	"strings"
)

var (
	// markdown headline pattern
	anyLevelMarkdownHeadline = regexp.MustCompile(`^(#+?)([^#].+?[^#])(#*)$`)
)

// Parse an item with a title, description and content
func Parse(item *repository.Item, lines []string, fallbackTitle string) (sucess bool, err error) {

	// parse the document
	if success, err := document.Parse(item, lines, fallbackTitle); !success {
		return sucess, err
	}

	// split the lines again
	presentationLines := make([]string, 0)
	lines = strings.Split(item.RawContent, "\n")

	// separate the slides with horizontal rule
	for lineNumber, line := range lines {

		// skip non-headlines
		if !isHeadline(line) {
			presentationLines = append(presentationLines, line)
			continue
		}

		// prepend a horizontal rule if
		// - its not the first line
		// - the headline is not already preceeded with a horizontal rule
		if lineNumber > 0 && !horizontalRuleAlreadyPresentIn(lines[0:lineNumber-1]) {

			presentationLines = append(presentationLines, "")
			presentationLines = append(presentationLines, "---")
			presentationLines = append(presentationLines, "")

		}

		// Fix the headline levels:
		// If the current line is followed by content make the current headline a level-two headline.
		// If the current line is not followed by content make the current headline a level-one headline.
		if lineNumber < len(lines)-1 && followingLinesContainContent(lines[lineNumber+1:]) {

			// slide with content -> h2 headline
			secondLevelHeadline := anyLevelMarkdownHeadline.ReplaceAllString(line, "## $2")
			presentationLines = append(presentationLines, secondLevelHeadline)

		} else {

			// slide without content -> h1 headline
			firstLevelHeadline := anyLevelMarkdownHeadline.ReplaceAllString(line, "# $2")
			presentationLines = append(presentationLines, firstLevelHeadline)

		}
	}

	// save the presentation code
	item.RawContent = strings.TrimSpace(strings.Join(presentationLines, "\n"))

	return true, nil
}

// Determine whether the supplied text
// is a markdown headline.
func isHeadline(text string) bool {
	return strings.HasPrefix(text, "#")
}

// Determine whether the supplied lines contain a horizontal rule
// before a line contains actual content.
func horizontalRuleAlreadyPresentIn(lines []string) bool {
	if len(lines) == 0 {
		return false
	}

	for lineNumber := len(lines) - 1; lineNumber >= 0; lineNumber-- {
		line := lines[lineNumber]

		if pattern.EmptyLinePattern.MatchString(line) {
			continue
		}

		return pattern.HorizontalRulePattern.MatchString(line)
	}

	panic("Unreachable")
}

// Determine if the supplied lines contain content before
// the next slide-end (horizontal rule or headine).
func followingLinesContainContent(lines []string) bool {
	if len(lines) == 0 {
		return false
	}

	for _, line := range lines {

		// an empty line is not content.
		if pattern.EmptyLinePattern.MatchString(line) {
			continue
		}

		// if there is another headline, there is no more content.
		if isHeadline(line) {
			return false
		}

		// if there is a horizontal rule, there is no more content.
		if pattern.HorizontalRulePattern.MatchString(line) {
			return false
		}

		// if it is not white-space, a headline or a
		// horizontal rule it must be content.
		return true
	}

	panic("Unreachable")
}
{'fraction_non_alphanumeric': 0.10649580203269995, 'fraction_numerical': 0.005155398438650758, 'mean_word_length': 3.9025270758122743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4805367', 'n_tokens_mistral': 2231, 'n_tokens_neox': 1956, 'n_words': 631}