<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ete3 import TreeNode

from pathlib import Path
import json
import copy
from datetime import datetime, timedelta
import uuid
from urllib.parse import urlparse, unquote_plus
from base64 import b64decode
from collections import defaultdict
import logging
import re
import os
from io import BytesIO
import hashlib
from operator import itemgetter
from bs4 import BeautifulSoup
import html


class Har2TreeError(Exception):
    def __init__(self, message):
        super(Har2TreeError, self).__init__(message)
        self.message = message


def rebuild_url(base_url, partial, known_urls):
    splitted_base_url = urlparse(base_url)
    # Remove all possible quotes
    partial = partial.strip()
    partial = html.unescape(partial)
    partial = unquote_plus(partial)
    if not partial:
        return ''
    if re.match('^https?://', partial):
        # we have a proper URL... hopefully
        # DO NOT REMOVE THIS CLAUSE, required to make the difference with a path
        final_url = partial
    elif partial.startswith('//'):
        # URL without scheme => takes the scheme from the caller
        final_url = f'{splitted_base_url.scheme}:{partial}'
        if final_url not in known_urls:
            logging.info(f'URL without scheme: {base_url} - {partial} - {final_url}')
    elif partial.startswith('/') or partial[0] not in [';', '?', '#']:
        # We have a path
        if partial[0] != '/':
            # Yeah, that happens, and the browser appends the path in redirect_url to the current path
            if base_url[-1] == '/':
                # Example: http://foo.bar/some/path/ and some/redirect.html becomes http://foo.bar/some/path/some/redirect.html
                final_url = f'{base_url}{partial}'
            else:
                # Need to strip the last part of the URL down to the first / (included), and attach the redirect
                # Example: http://foo.bar/some/path/blah and some/redirect.html becomes http://foo.bar/some/path/some/redirect.html
                last_slash = base_url.rfind('/') + 1
                final_url = f'{base_url[:last_slash]}{partial}'
        else:
            final_url = f'{splitted_base_url.scheme}://{splitted_base_url.netloc}{partial}'
        if final_url not in known_urls:
            # There is something weird, to investigate
            logging.info(f'URL without netloc: {base_url} - {partial} - {final_url}')
    elif partial.startswith(';'):
        # URL starts at the parameters
        final_url = '{}{}'.format(base_url.split(';')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only parameter: {base_url} - {partial} - {final_url}')
    elif partial.startswith('?'):
        # URL starts at the query
        final_url = '{}{}'.format(base_url.split('?')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only query: {base_url} - {partial} - {final_url}')
    elif partial.startswith('#'):
        # URL starts at the fragment
        final_url = '{}{}'.format(base_url.split('#')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only fragment: {base_url} - {partial} - {final_url}')

    if final_url not in known_urls:
        # sometimes, the port is in the redirect, and striped later on...
        if final_url.startswith('https://') and ':443' in final_url:
            final_url = final_url.replace(':443', '')
        if final_url.startswith('http://') and ':80' in final_url:
            final_url = final_url.replace(':80', '')

    if final_url not in known_urls:
        # strip the single-dot crap: https://foo.bar/path/./blah.js => https://foo.bar/path/blah.js
        try:
            parsed = urlparse(final_url)
            final_url = parsed._replace(path=str(Path(parsed.path).resolve())).geturl()
        except Exception:
            logging.info(f'Not a URL: {base_url} - {partial}')

    if final_url not in known_urls and final_url + '/' in known_urls:
        # last thing I can think of
        final_url = f'{final_url}/'

    return final_url


# Standalone methods to extract and cleanup content from an HTML blob.
def url_cleanup(dict_to_clean, base_url, all_requests):
    to_return = {}
    for key, urls in dict_to_clean.items():
        to_return[key] = []
        for url in urls:
            if url.startswith('data'):
                # print(html_doc.getvalue())
                continue
            to_attach = url.strip()
            if to_attach.startswith("\\'") or to_attach.startswith('\\"'):
                to_attach = to_attach[2:-2]
            if to_attach.startswith('\'') or to_attach.startswith('"'):
                to_attach = to_attach[1:-1]
            to_attach = rebuild_url(base_url, to_attach, all_requests)
            if to_attach.startswith('http'):
                to_return[key].append(to_attach)
            else:<commit_msg>fix: Properly handle URLs with no path<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ete3 import TreeNode

from pathlib import Path
import json
import copy
from datetime import datetime, timedelta
import uuid
from urllib.parse import urlparse, unquote_plus
from base64 import b64decode
from collections import defaultdict
import logging
import re
import os
from io import BytesIO
import hashlib
from operator import itemgetter
from bs4 import BeautifulSoup
import html


class Har2TreeError(Exception):
    def __init__(self, message):
        super(Har2TreeError, self).__init__(message)
        self.message = message


def rebuild_url(base_url, partial, known_urls):
    splitted_base_url = urlparse(base_url)
    # Remove all possible quotes
    partial = partial.strip()
    partial = html.unescape(partial)
    partial = unquote_plus(partial)
    if not partial:
        return ''
    if re.match('^https?://', partial):
        # we have a proper URL... hopefully
        # DO NOT REMOVE THIS CLAUSE, required to make the difference with a path
        final_url = partial
    elif partial.startswith('//'):
        # URL without scheme => takes the scheme from the caller
        final_url = f'{splitted_base_url.scheme}:{partial}'
        if final_url not in known_urls:
            logging.info(f'URL without scheme: {base_url} - {partial} - {final_url}')
    elif partial.startswith('/') or partial[0] not in [';', '?', '#']:
        # We have a path
        if partial[0] != '/':
            # Yeah, that happens, and the browser appends the path in redirect_url to the current path
            if base_url[-1] == '/':
                # Example: http://foo.bar/some/path/ and some/redirect.html becomes http://foo.bar/some/path/some/redirect.html
                final_url = f'{base_url}{partial}'
            else:
                # Need to strip the last part of the URL down to the first / (included), and attach the redirect
                # Example: http://foo.bar/some/path/blah and some/redirect.html becomes http://foo.bar/some/path/some/redirect.html
                last_slash = base_url.rfind('/') + 1
                final_url = f'{base_url[:last_slash]}{partial}'
        else:
            final_url = f'{splitted_base_url.scheme}://{splitted_base_url.netloc}{partial}'
        if final_url not in known_urls:
            # There is something weird, to investigate
            logging.info(f'URL without netloc: {base_url} - {partial} - {final_url}')
    elif partial.startswith(';'):
        # URL starts at the parameters
        final_url = '{}{}'.format(base_url.split(';')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only parameter: {base_url} - {partial} - {final_url}')
    elif partial.startswith('?'):
        # URL starts at the query
        final_url = '{}{}'.format(base_url.split('?')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only query: {base_url} - {partial} - {final_url}')
    elif partial.startswith('#'):
        # URL starts at the fragment
        final_url = '{}{}'.format(base_url.split('#')[0], partial)
        if final_url not in known_urls:
            logging.info(f'URL with only fragment: {base_url} - {partial} - {final_url}')

    if final_url not in known_urls:
        # sometimes, the port is in the redirect, and striped later on...
        if final_url.startswith('https://') and ':443' in final_url:
            final_url = final_url.replace(':443', '')
        if final_url.startswith('http://') and ':80' in final_url:
            final_url = final_url.replace(':80', '')

    if final_url not in known_urls:
        # strip the single-dot crap: https://foo.bar/path/./blah.js => https://foo.bar/path/blah.js
        try:
            parsed = urlparse(final_url)
            if parsed.path:
                # NOTE: Path('').resolve() => PosixPath('/path/to/current/directory') <= if you run that from your home dir, it is the path to your home dir
                # FIXME: Path('sdsfs/../dsfsdfs/../..').resolve() => PosixPath('/path/to/current')
                final_url = parsed._replace(path=str(Path(parsed.path).resolve())).geturl()
            else:
                final_url = parsed._replace(path='/').geturl()
        except Exception:
            logging.info(f'Not a URL: {base_url} - {partial}')

    if final_url not in known_urls and final_url + '/' in known_urls:
        # last thing I can think of
        final_url = f'{final_url}/'

    return final_url


# Standalone methods to extract and cleanup content from an HTML blob.
def url_cleanup(dict_to_clean, base_url, all_requests):
    to_return = {}
    for key, urls in dict_to_clean.items():
        to_return[key] = []
        for url in urls:
            if url.startswith('data'):
                # print(html_doc.getvalue())
                continue
            to_attach = url.strip()
            if to_attach.startswith("\\'") or to_attach.startswith('\\"'):
                to_attach = to_attach[2:-2]
            if to_attach.startswith('\'') or to_attach.startswith('"'):
                to_attach = to_attach[1:-1]
            to_attach = rebuild_url(base_url, to_attach, all_requests)
            if to_attach.startswith('http'):
                to_return[key].append(to_attach)
            else:
{'fraction_non_alphanumeric': 0.12173155142359093, 'fraction_numerical': 0.005810575246949448, 'mean_word_length': 2.4782755136409564, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1955366', 'n_tokens_mistral': 3141, 'n_tokens_neox': 3092, 'n_words': 860}