<commit_before>from bs4 import BeautifulSoup as BS
import requests
from urlparse import urlparse

class WebPage(object):

  def __init__(self, url):
    self.page = requests.get(url) 
    self.soup = BS(self.page.text, "html.parser")

  def get_assets(self):
    assets = []
    assets += self.get_javascript()
    assets += self.get_images()
    assets += self.get_links()
    assets += self.get_anchors()
    return assets

  def __resource(self, element, attribute):
    resources = []
    resources += self.soup.find_all(element)
    url_strings = filter(lambda val: val is not None, map(lambda res: res.get(attribute), resources))
    urls = map(lambda url_str: urlparse(url_str), url_strings)
    return urls

  def get_javascript(self):
    return self.__resource('script', 'src')

  def get_images(self):
    return self.__resource('img', 'src')

  def get_links(self):
    return self.__resource('link', 'href')

  def get_anchors(self):
    return self.__resource('a', 'href')
<commit_msg>convert relative to absolute url<commit_after>import os
from bs4 import BeautifulSoup as BS
import requests
from urlparse import urlparse
from urlparse import urljoin

class WebPage(object):

  def __init__(self, url):
    self.url = urlparse(url)
    self.page = requests.get(url) 
    self.soup = BS(self.page.text, "html.parser")

  def get_assets(self):
    assets = []
    assets += self.get_js()
    assets += self.get_images()
    assets += self.get_links()
    assets += self.get_anchors()
    return assets

  def __resource(self, element, attribute):
    resources = []
    resources += self.soup.find_all(element)
    url_strings = filter(lambda val: val is not None, map(lambda res: res.get(attribute), resources))
    
    # parse url as well as convert relative url to absolute url
    def parse_url(url_str):
      # TODO handle tel:11234567890
      absolute_url = urljoin(self.page.url, url_str) 
      if os.environ['DEBUG'] and url_str != absolute_url:
        print "converted: ", url_str, " --> ", absolute_url
      return urlparse(absolute_url) 

    urls = map(parse_url, url_strings)
    return urls

  def get_js(self):
    return self.__resource('script', 'src')

  def get_images(self):
    return self.__resource('img', 'src')

  def get_links(self):
    return self.__resource('link', 'href')

  def get_anchors(self):
    return self.__resource('a', 'href')

{'fraction_non_alphanumeric': 0.12144053601340034, 'fraction_numerical': 0.005443886097152429, 'mean_word_length': 3.684313725490196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6106334', 'n_tokens_mistral': 771, 'n_tokens_neox': 743, 'n_words': 200}