<commit_before>package com.dua3.utility.text;

import org.commonmark.node.Node;
import org.commonmark.parser.Parser;
import org.commonmark.renderer.html.HtmlRenderer;

public class MarkDownUtil {

    public static RichText toRichText(String source) {
        Parser parser = Parser.builder().build();
        Node node = parser.parse(source);

        RichTextRenderer renderer = new RichTextRenderer();
        return renderer.render(node);
    }

    public static String toHTML(String source) {
        Parser parser = Parser.builder().build();
        Node node = parser.parse(source);

        HtmlRenderer renderer = HtmlRenderer.builder().build();
        return renderer.render(node);
    }

    private MarkDownUtil() {
        // utility class
    }

}<commit_msg>loading from URL, resolve links<commit_after>package com.dua3.utility.text;

import java.io.BufferedReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;

import org.commonmark.node.Image;
import org.commonmark.node.Node;
import org.commonmark.parser.Parser;
import org.commonmark.parser.PostProcessor;
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.html.AttributeProvider;
import org.commonmark.renderer.html.AttributeProviderContext;
import org.commonmark.renderer.html.AttributeProviderFactory;
import org.commonmark.renderer.html.HtmlNodeRendererContext;
import org.commonmark.renderer.html.HtmlNodeRendererFactory;
import org.commonmark.renderer.html.HtmlRenderer;
import org.commonmark.renderer.html.HtmlRenderer.Builder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class MarkDownUtil {
	private static final Logger LOG = LoggerFactory.getLogger(MarkDownUtil.class);

	public static RichText toRichText(String source) {
		Parser parser = Parser.builder().build();
		Node node = parser.parse(source);

		RichTextRenderer renderer = new RichTextRenderer();
		return renderer.render(node);
	}

	public static String toHTML(String source) {
		Parser parser = Parser.builder().build();
		Node node = parser.parse(source);

		HtmlRenderer renderer = HtmlRenderer.builder().build();
		return renderer.render(node);
	}

	static class LinkResolvingAttributeProvider implements AttributeProvider {
		private final URI baseUri;

		public LinkResolvingAttributeProvider(URI baseUri) {
			this.baseUri = baseUri;
		}

		@Override
		public void setAttributes(Node node, String tagName, Map<String, String> attributes) {
			if (node instanceof Image) {
				attributes.computeIfPresent("src", (k, v) -> {
					try {
						return baseUri.resolve(new URI(v)).toString();
					} catch (URISyntaxException e) {
						LOG.warn("could not parse src attribute: ", v);
						return v;
					}
				});
			}
		}
	}

	public static String toHTML(URL url, boolean resolveLinks) throws IOException {
		try {
			URI uri = url.toURI();
			try (BufferedReader reader = Files.newBufferedReader(Paths.get(uri), StandardCharsets.UTF_8)) {
				Parser parser = Parser.builder().build();
				Node node = parser.parseReader(reader);

				Builder rendererBuilder = HtmlRenderer.builder();

				if (resolveLinks) {
					rendererBuilder = rendererBuilder.attributeProviderFactory(new AttributeProviderFactory() {
						@Override
						public AttributeProvider create(AttributeProviderContext context) {
							return new LinkResolvingAttributeProvider(uri);
						}
					});
				}

				HtmlRenderer renderer = rendererBuilder.build();
				return renderer.render(node);
			}
		} catch (URISyntaxException e) {
			throw new IOException("could not convert URI to URL: " + url, e);
		}
	}

	private MarkDownUtil() {
		// utility class
	}

}
{'fraction_non_alphanumeric': 0.10436382754994743, 'fraction_numerical': 0.0013144058885383807, 'mean_word_length': 4.141891891891892, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4149069', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1088, 'n_words': 250}