<commit_before>import urllib3, re, pdb
from urllib.parse import urlparse
from bs4 import BeautifulSoup

http = urllib3.PoolManager()

def iterlen(itr):
    l = 0
    for i in itr:
        l += 1
    return l
def firstMatchingID(soup, reg):
    def recur(soup, reg, ans):
        print(type(soup))
        if soup.has_attr('id') and (not re.match(reg, soup.attrs["id"]) == None):
            return soup
        if iterlen(soup.children) == 0:
            return []
        for i in soup.children:

            c = recur(i, reg, ans)
            if not c == []:
                return c
        return []
    recur(soup, reg, [])

def containsDate(strng):
    if isinstance(strng, list):
        return any(map(lambda x: containsDate(x), strng))
    match = False
    if isinstance(strng, str):
        strng = strng.lower()
        for i in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
                  'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'november', 'december']:
            match = match or not strng.find(i) == -1
    return match
def dictContainsDate(dic):
    match = False
    for i in dic.values():
        match = match or containsDate(i)
    return match
def isRelativeLink(st):
    return urlparse(st).netloc == '' and not urlparse(st).path == ''
def isAbsoluteLink(st):
    return not urlparse(st).netloc == ''

def scrape3(url):
    ll = set([])
    r = http.request('GET', url)
    soup = BeautifulSoup(r.data)
    #looking for anything with the header matching re archive
    archive = soup.find_all(id=re.compile("archive", re.I))
    if not len(archive) == 0:
        for i in map(lambda x: x.find_all('a', href=True), archive):
            for j in i:
                ll.add(j)
    links = soup.find_all('a', href=True)
    for i in filter(lambda x: containsDate(x.contents) or dictContainsDate(x.attrs), links):
        ll.add(i)
    def same(x):
        try:
            return x.attrs['href'][0:len(url)] == url
        except:
            return False

    ll = list(filter(same, ll))
    return ll

def pl(h):
    for i in h:
        print(i.attrs['href'])<commit_msg>Passes adequately a lot of tests, but still not perfect. Implemented beginnings of machine learning for classifying blog posts.<commit_after>import urllib3, re, pdb, arrow, itertools
from urllib.parse import urlparse
from bs4 import BeautifulSoup

def powerset(iterable):
    "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"
    s = list(iterable)
    return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))

http = urllib3.PoolManager()

def iterlen(itr):
    l = 0
    for i in itr:
        l += 1
    return l
def firstMatchingID(soup, reg):
    def recur(soup, reg, ans):
        print(type(soup))
        if soup.has_attr('id') and (not re.match(reg, soup.attrs["id"]) == None):
            return soup
        if iterlen(soup.children) == 0:
            return []
        for i in soup.children:

            c = recur(i, reg, ans)
            if not c == []:
                return c
        return []
    recur(soup, reg, [])

def containsDate(strng):
    if isinstance(strng, list):
        return any(map(lambda x: containsDate(x), strng))
    if isinstance(strng, str):
        template = ["YYYY-MM-DD","YYYY-M-DD", "YYYY-MM-D", "YYYY-M-D", "dddd-MMM-YYYY",
                    "dddd-MMMM-YYYY", "MMMM-dddd-YYYY", "MMMM-DD-YYYY", "YYYY-MMMM", "YYYY-MMM", "YYYY-MM",
                    'YYYY', 'MMMM', "MMM"]
        for i in template:
            try:
                arrow.get(strng, i)
                return True
            except Exception:
                pass
    return False
def dictContainsDate(dic):
    match = False
    for i in dic.values():
        match = match or containsDate(i)
    return match
def isRelativeLink(st):
    return urlparse(st).netloc == '' and not urlparse(st).path == ''
def isAbsoluteLink(st):
    return not urlparse(st).netloc == ''
def tests():
    urls = [("http://freegamer.blogspot.com/",),
            ("http://www.mode7games.com/blog/",),
            ("http://xkcd.com/archive/",),
            "http://www.joelonsoftware.com/backIssues-2000-03.html"]

class Feature():
    number = 0
    word = 1
    other = 2
def classify(chunk):
    if chunk.isalpha():
        return Feature.word
    elif chunk.isdigit():
        return Feature.number
    return Feature.other
def freq(array, num):
    return sum([1 for i in array if i == num])
def feature(url):
    fv = re.split(re.compile('[-_/.//]'), url)
    fv = filter(lambda x: not x == '', fv)
    fv = map(lambda x: classify(x), fv)
    return {'number': freq(fv, Feature.number),
            'word': freq(fv, Feature.word),
            'other': freq(fv, Feature.other),
            'url': url}

def scrape3(url):
    arst = set([])
    r = http.request('GET', url)
    soup = BeautifulSoup(r.data)
    #looking for anything with the header matching re archive
    archive = soup.find_all(id=re.compile("archive", re.I))
    if not len(archive) == 0:
        for i in map(lambda x: x.find_all('a', href=True), archive):
            for j in i:
                arst.add(j)
    #find all things with dates in content
    links = soup.find_all('a', href=True)
    for i in filter(lambda x: containsDate(x.contents) or dictContainsDate(x.attrs), links):
        arst.add(i)
    #make sure it comes from same url
    def same(x):
        try:
            att = x.attrs['href']
            return not arrow.get(att) == None or att[0:len(url)] == url or att[0] == '/'
        except Exception:
            return False
    arst = list(filter(same, arst))

    return arst

def pl(h):
    for i in h:
        print(i.attrs['href'])
{'fraction_non_alphanumeric': 0.14721254355400698, 'fraction_numerical': 0.008188153310104529, 'mean_word_length': 2.314665127020785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581726', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1906, 'n_words': 537}