<commit_before>#!/usr/bin/env python3

import urllib.request, lxml.html

YOUTUBE_BASE_URL = 'https://www.youtube.com'
YOUTUBE_SEARCH_URL = YOUTUBE_BASE_URL + '/results?search_query={0}' 
XPATH_SEARCH_RESULTS = '//a[contains(@class, "yt-uix-tile-link")]'

def html_get(url):
    obj = urllib.request.urlopen(url)
    return lxml.html.parse(obj)

def search(string):
    string = urllib.parse.quote_plus(string)        # escape the string :)
    url = YOUTUBE_SEARCH_URL.format(string)         # get search url
    obj = html_get(url)
    global results
    results = []
    for item in obj.xpath(XPATH_SEARCH_RESULTS):
        res = {}
        res['title'] = item.get('title')
        res['url'] = YOUTUBE_BASE_URL + item.get('href')
        try:
            res['id'] = _get_id(item.get('href'))
            results.append(res)
        except KeyError:pass
            #print("[WARNING] Could not parse url {0}, skipping...".format(item.get('href')))
    return results

def _get_id(url):
    query = urllib.parse.urlsplit(url).query
    return urllib.parse.parse_qs(query)['v'][0]

def search1(string):
    return search(string)[0]<commit_msg>Added uniform representation for download instances<commit_after>#!/usr/bin/env python3

import urllib.request, lxml.html

YOUTUBE_BASE_URL = 'https://www.youtube.com'
YOUTUBE_SEARCH_URL = YOUTUBE_BASE_URL + '/results?search_query={0}' 
XPATH_SEARCH_RESULTS = '//a[contains(@class, "yt-uix-tile-link")]'

def html_get(url):
    obj = urllib.request.urlopen(url)
    return lxml.html.parse(obj)

def search(string):
    string = urllib.parse.quote_plus(string)        # escape the string :)
    url = YOUTUBE_SEARCH_URL.format(string)         # get search url
    obj = html_get(url)
    global results
    results = []
    for item in obj.xpath(XPATH_SEARCH_RESULTS):
        res = {}
        res['title'] = item.get('title')
        res['url'] = YOUTUBE_BASE_URL + item.get('href')
        try:
            res['id'] = _get_id(item.get('href'))
            results.append(res)
        except KeyError:pass
            #print("[WARNING] Could not parse url {0}, skipping...".format(item.get('href')))
    return results

def _get_id(url):
    query = urllib.parse.urlsplit(url).query
    return urllib.parse.parse_qs(query)['v'][0]

def search1(string):
    return search(string)[0]

def tostring(item):
    return "{0} [{1}]".format(item['title'], item['id'])
{'fraction_non_alphanumeric': 0.1509983361064892, 'fraction_numerical': 0.005823627287853577, 'mean_word_length': 3.2641843971631204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3445989', 'n_tokens_mistral': 847, 'n_tokens_neox': 844, 'n_words': 152}