<commit_before>from bs4 import BeautifulSoup
import bs4
import requests
import re
import sys
import json
from urlparse import urljoin, urlparse
from Queue import Queue

email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
phone_regex = re.compile(r"(?:(?:\+?1\s*(?:[.-]\s*)?)?(?:\(\s*([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\s*\)|([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\s*(?:[.-]\s*)?)?([2-9]1[02-9]|[2-9][02-9]1|[2-9][02-9]{2})\s*(?:[.-]\s*)?([0-9]{4})(?:\s*(?:#|x\.?|ext\.?|extension)\s*(\d+))?")

class Webpage:
    def __init__(self, html, url):
        self.html = html
        self.url = url

    def parse(self):
        soup = BeautifulSoup(self.html, "lxml")

        strings = [string for string in soup.strings]

        # find links
        _found_links = []
        def merge(url):
            _found_links.append(url)
            return urljoin(self.url, url)
        links = [{"url": merge(a['href']), "name": _scrub(a.text) if _scrub(a.text) else a['href']} for a in soup.find_all("a") if a.get('href') is not None and a.get('href') is not "#" and a.get("href").startswith("mailto:") is not True and a.get("href") not in _found_links]

        # find social media
        social_media = {}
        _social_media_sites = ["facebook.com", "youtube.com", "twitter.com", "linkedin.com", "github.com", "plus.google.com"]
        _social_media_urls = []
        for link in links:
            for site in _social_media_sites:
                if site in link['url'].lower() and link['url'] not in _social_media_urls:
                    if not (site == "twitter.com" and "/intent/" in link['url']):
                        if site not in social_media:
                            social_media[site] = []
                        social_media[site].append(link)
                        _social_media_urls.append(link['url'])
        del _social_media_sites, _social_media_urls

        # find description
        description = (soup.find('meta', attrs={'name':'og:description'}) or soup.find('meta', attrs={'property':'description'}) or soup.find('meta', attrs={'name':'description'}))
        if description is not None:
            description = description.get("content")

        # find telephone numbers
        telephones = []
        i = 0
        for string in strings:
            for match in phone_regex.finditer(string):
                extended = _get_desc_phone(strings, i)
                number = match.group(0)
                if len(match.string) > 100: # or _alpha_ratio(match.string) > 0.4:
                    break
                if ("EIN" in match.string or "EIN" in extended) and ("tax" in match.string or "tax" in extended):
                    continue
                if extended == match.string:
                    if not len(_alpha(extended.replace(number, "")).strip()) > 0:
                        extended = None
                if extended.endswith(number):
                    extended = extended[:-(len(number))].strip()
                if match.string is None:
                    continue
                telephones.append({
                    "number": number,
                    "extended": extended
                })
                break<commit_msg>nullsafe<commit_after>from bs4 import BeautifulSoup
import bs4
import requests
import re
import sys
import json
from urlparse import urljoin, urlparse
from Queue import Queue

email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
phone_regex = re.compile(r"(?:(?:\+?1\s*(?:[.-]\s*)?)?(?:\(\s*([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9])\s*\)|([2-9]1[02-9]|[2-9][02-8]1|[2-9][02-8][02-9]))\s*(?:[.-]\s*)?)?([2-9]1[02-9]|[2-9][02-9]1|[2-9][02-9]{2})\s*(?:[.-]\s*)?([0-9]{4})(?:\s*(?:#|x\.?|ext\.?|extension)\s*(\d+))?")

class Webpage:
    def __init__(self, html, url):
        self.html = html
        self.url = url

    def parse(self):
        soup = BeautifulSoup(self.html, "lxml")

        strings = [string for string in soup.strings]

        # find links
        _found_links = []
        def merge(url):
            _found_links.append(url)
            return urljoin(self.url, url)
        links = [{"url": merge(a['href']), "name": _scrub(a.text) if _scrub(a.text) else a['href']} for a in soup.find_all("a") if a.get('href') is not None and a.get('href') is not "#" and a.get("href").startswith("mailto:") is not True and a.get("href") not in _found_links]

        # find social media
        social_media = {}
        _social_media_sites = ["facebook.com", "youtube.com", "twitter.com", "linkedin.com", "github.com", "plus.google.com"]
        _social_media_urls = []
        for link in links:
            for site in _social_media_sites:
                if site in link['url'].lower() and link['url'] not in _social_media_urls:
                    if not (site == "twitter.com" and "/intent/" in link['url']):
                        if site not in social_media:
                            social_media[site] = []
                        social_media[site].append(link)
                        _social_media_urls.append(link['url'])
        del _social_media_sites, _social_media_urls

        # find description
        description = (soup.find('meta', attrs={'name':'og:description'}) or soup.find('meta', attrs={'property':'description'}) or soup.find('meta', attrs={'name':'description'}))
        if description is not None:
            description = description.get("content")

        # find telephone numbers
        telephones = []
        i = 0
        for string in strings:
            for match in phone_regex.finditer(string):
                extended = _get_desc_phone(strings, i)
                number = match.group(0)
                if len(match.string) > 100: # or _alpha_ratio(match.string) > 0.4:
                    break
                if ("EIN" in match.string or "EIN" in extended) and ("tax" in match.string or "tax" in extended):
                    continue
                if extended == match.string:
                    if not len(_alpha(extended.replace(number, "")).strip()) > 0:
                        extended = None
                    elif extended.endswith(number):
                        extended = extended[:-(len(number))].strip()
                if match.string is None:
                    continue
                telephones.append({
                    "number": number,
                    "extended": extended
                })
                break
{'fraction_non_alphanumeric': 0.17018652689995376, 'fraction_numerical': 0.02404809619238477, 'mean_word_length': 2.013469577333953, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4469598', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2033, 'n_words': 445}