<commit_before>package de.unitrier.st.soposthistory.urls;

import de.unitrier.st.util.Patterns;

import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

public class Link {
    String fullMatch;
    String anchor; // the link anchor visible to the user
    String reference; // internal Markdown reference for the link
    String url;
    String title;
    String protocol;
    String completeDomain;
    String rootDomain;
    String path;

    public String getFullMatch() {
        return fullMatch;
    }

    public String getAnchor() {
        return anchor;
    }

    public String getReference() {
        return reference;
    }

    public String getUrl() {
        return url;
    }

    public String getTitle() {
        return title;
    }

    public String getProtocol() {
        return protocol;
    }

    public String getCompleteDomain() {
        return completeDomain;
    }

    public String getRootDomain() {
        return rootDomain;
    }

    public String getPath() {
        return path;
    }

    public void extractURLComponents() {
        // protocol
        Matcher protocolMatcher = Patterns.protocol.matcher(url);
        if (protocolMatcher.find()) {
            this.protocol = protocolMatcher.group(1);
        } else {
            throw new IllegalArgumentException("Extraction of protocol failed for URL: " + url);
        }

        // domain
        Matcher completeDomainMatcher = Patterns.completeDomain.matcher(url);
        if (completeDomainMatcher.find()) {
            this.completeDomain = completeDomainMatcher.group(1);
        } else {
            throw new IllegalArgumentException("Extraction of complete domain failed for URL: " + url);
        }
        Matcher rootDomainMatcher = Patterns.rootDomain.matcher(completeDomain);
        if (rootDomainMatcher.find()) {
            this.rootDomain = rootDomainMatcher.group(1);
        } else {
            throw new IllegalArgumentException("Extraction of root domain failed for URL: " + url);
        }

        // path
        Matcher pathMatcher = Patterns.path.matcher(url);
        if (pathMatcher.find()) {
            this.path = pathMatcher.group(1);
            // remove trailing slash
            if (this.path.endsWith("/")) {
                this.path = this.path.substring(0, this.path.length()-1);
            }
        }
    }

    public static List<Link> extractBare(String markdownContent) {
        LinkedList<Link> extractedLinks = new LinkedList<>();
        Matcher urlMatcher = Patterns.url.matcher(markdownContent);

        while (urlMatcher.find()) {
            Link extractedLink = new Link();
            extractedLink.fullMatch = urlMatcher.group(0).trim();
            extractedLink.url = extractedLink.fullMatch;  // for bare links, the full match is equal to the url match
            if (extractedLink.url.length() > 0) {
                extractedLink.extractURLComponents();
                extractedLinks.add(extractedLink);
            }
        }
<commit_msg>Also move extraction methods to util project<commit_after>package de.unitrier.st.soposthistory.urls;

import de.unitrier.st.util.Patterns;

import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.stream.Collectors;

public class Link {
    String fullMatch;
    String anchor; // the link anchor visible to the user
    String reference; // internal Markdown reference for the link
    String url;
    String title;
    String protocol;
    String completeDomain;
    String rootDomain;
    String path;

    public String getFullMatch() {
        return fullMatch;
    }

    public String getAnchor() {
        return anchor;
    }

    public String getReference() {
        return reference;
    }

    public String getUrl() {
        return url;
    }

    public String getTitle() {
        return title;
    }

    public String getProtocol() {
        return protocol;
    }

    public String getCompleteDomain() {
        return completeDomain;
    }

    public String getRootDomain() {
        return rootDomain;
    }

    public String getPath() {
        return path;
    }

    private void extractURLComponents() {
        this.protocol = Patterns.extractProtocol(url);
        this.completeDomain = Patterns.extractCompleteDomain(url);
        this.rootDomain = Patterns.extractRootDomain(completeDomain);
        this.path = Patterns.extractPath(url);
    }

    public static List<Link> extractBare(String markdownContent) {
        LinkedList<Link> extractedLinks = new LinkedList<>();
        Matcher urlMatcher = Patterns.url.matcher(markdownContent);

        while (urlMatcher.find()) {
            Link extractedLink = new Link();
            extractedLink.fullMatch = urlMatcher.group(0).trim();
            extractedLink.url = extractedLink.fullMatch;  // for bare links, the full match is equal to the url match
            if (extractedLink.url.length() > 0) {
                extractedLink.extractURLComponents();
                extractedLinks.add(extractedLink);
            }
        }

{'fraction_non_alphanumeric': 0.09085648148148148, 'fraction_numerical': 0.0019290123456790122, 'mean_word_length': 2.3152173913043477, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '363529', 'n_tokens_mistral': 1371, 'n_tokens_neox': 1333, 'n_words': 355}