<commit_before>package com.widowcrawler.parse;

import com.netflix.governator.annotations.AutoBindSingleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.core.UriBuilder;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author <NAME>
 */
@AutoBindSingleton
public class LinkNormalizer {

    private static final Logger logger = LoggerFactory.getLogger(LinkNormalizer.class);

    private static final String JS_PREFIX = "javascript:";

    /**
     * Normalizes a link extracted from a page given the page's original URI.
     *
     * @param original The URI of the page that is being parsed
     * @param extracted The extracted link from the page
     * @return The normalized URI, or null if there was an error parsing the extracted URI.
     */
    public String normalize(String original, String extracted) {

        Validate.notBlank(original);
        Validate.notNull(extracted);

        if (StringUtils.startsWith(extracted, JS_PREFIX)) {
            return null;
        }

        URI originalUri = null;
        URI extractedUri = null;

        try {
            originalUri = new URI(original);
            extractedUri = new URI(extracted);
        } catch (URISyntaxException ex) {
            logger.warn("Extracted URI is invalid: " + extracted, ex);
            return null;
        }

        UriBuilder retval = UriBuilder.fromUri(extractedUri);
        boolean hasHost = StringUtils.isNotBlank(extractedUri.getHost());
        boolean hasScheme = StringUtils.isNotBlank(extractedUri.getScheme());

        if (!hasScheme) {
            retval.scheme(originalUri.getScheme());<commit_msg>Removing "javascript:" detection from link normalization<commit_after>package com.widowcrawler.parse;

import com.netflix.governator.annotations.AutoBindSingleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.core.UriBuilder;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author <NAME>
 */
@AutoBindSingleton
public class LinkNormalizer {

    private static final Logger logger = LoggerFactory.getLogger(LinkNormalizer.class);

    /**
     * Normalizes a link extracted from a page given the page's original URI.
     *
     * @param original The URI of the page that is being parsed
     * @param extracted The extracted link from the page
     * @return The normalized URI, or null if there was an error parsing the extracted URI.
     */
    public String normalize(String original, String extracted) {

        Validate.notBlank(original);
        Validate.notNull(extracted);

        URI originalUri = null;
        URI extractedUri = null;

        try {
            originalUri = new URI(original);
            extractedUri = new URI(extracted);
        } catch (URISyntaxException ex) {
            logger.warn("Extracted URI is invalid: " + extracted, ex);
            return null;
        }

        UriBuilder retval = UriBuilder.fromUri(extractedUri);
        boolean hasHost = StringUtils.isNotBlank(extractedUri.getHost());
        boolean hasScheme = StringUtils.isNotBlank(extractedUri.getScheme());

        if (!hasScheme) {
            retval.scheme(originalUri.getScheme());
{'fraction_non_alphanumeric': 0.09010339734121123, 'fraction_numerical': 0.002363367799113737, 'mean_word_length': 3.0357568533969013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4627933', 'n_tokens_mistral': 940, 'n_tokens_neox': 923, 'n_words': 255}