<commit_before>const PROTOCOL_REGEX = new RegExp('^(?:[a-z]+:)?//', 'i');
const DOMAIN_REGEX = new RegExp('^(?:[a-z]+:)?//[^/]*', 'i');

export class UrlHelper {
    static join(...paths) {
        let len = paths.length - 1;
        return paths
            .filter((path) => !!path)
            .map((path, index) => {
                if (index === 0) {
                    return path.replace(/\/*$/, '');
                } else if (index === len) {
                    return path.replace(/^\/*/, '');
                }
                return path.replace(/^\/*/, '').replace(/\/*$/, '');
            })
            .join('/');
    }

    static isAbsoluteUrl(url) {
        return PROTOCOL_REGEX.test(url);
    }

    static resolve(base, relative) {
        if (!this.isAbsoluteUrl(base)) {
            throw new Error('base url is not an absolute url');
        }
        if (relative[0] === '/') {
            base = `${base.match(DOMAIN_REGEX)[0]}/`;
        }
        let stack = base.split('/');
        let parts = relative.split('/').filter((part) => part !== '');
        stack.pop();
        for (let i = 0; i < parts.length; i++) {
            if (parts[i] === '.') {
                continue;
            } else if (parts[i] === '..') {<commit_msg>fix: improve url resolution<commit_after>const PROTOCOL_REGEX = new RegExp('^(?:[a-z]+:)?//', 'i');
const DOMAIN_REGEX = new RegExp('^(?:[a-z]+:)?//[^/]*', 'i');
const DATA_REGEX = /^data:/;

export class UrlHelper {
    static join(...paths) {
        let len = paths.length - 1;
        return paths
            .filter((path) => !!path)
            .map((path, index) => {
                if (index === 0) {
                    return path.replace(/\/*$/, '');
                } else if (index === len) {
                    return path.replace(/^\/*/, '');
                }
                return path.replace(/^\/*/, '').replace(/\/*$/, '');
            })
            .join('/');
    }

    static isAbsoluteUrl(url) {
        return PROTOCOL_REGEX.test(url);
    }

    static isDataUrl(url) {
        return DATA_REGEX.test(url);
    }

    static resolve(base, relative) {
        if (relative[0] === '/') {
            if (!this.isAbsoluteUrl(base)) {
                throw new Error('base url is not an absolute url');
            }
            base = `${base.match(DOMAIN_REGEX)[0]}/`;
        }
        let stack = base.split('/');
        let parts = relative.split('/').filter((part) => part !== '');
        stack.pop();
        for (let i = 0; i < parts.length; i++) {
            if (parts[i] === '.') {
                continue;
            } else if (parts[i] === '..') {
{'fraction_non_alphanumeric': 0.21038765525028227, 'fraction_numerical': 0.003763643206624012, 'mean_word_length': 1.5170454545454546, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4075409', 'n_tokens_mistral': 841, 'n_tokens_neox': 815, 'n_words': 176}