<commit_before>from __future__ import unicode_literals

from builtins import str
import re
import time
from calendar import timegm
from util import hook, http, timesince


youtube_re = (r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)'
              '([-_a-z0-9]+)', re.I)

BASE_URL = 'https://www.googleapis.com/youtube/v3/'
INFO_URL = BASE_URL + 'videos?part=snippet,contentDetails,statistics&hl=en'
SEARCH_API_URL = BASE_URL + 'search'
VIDEO_URL = 'https://youtu.be/%s'


def get_video_description(vid_id, api_key):
    j = http.get_json(INFO_URL, id=vid_id, key=api_key)

    if not j['pageInfo']['totalResults']:
        return

    j = j['items'][0]

    duration = j['contentDetails']['duration'].replace('PT', '').lower()

    published = j['snippet']['publishedAt'].replace('.000Z', 'Z')
    published = time.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
    published = time.strftime("%Y.%m.%d", published)

    views = group_int_digits(j['statistics']['viewCount'], ',')
    likes = j['statistics'].get('likeCount', 0)
    dislikes = j['statistics'].get('dislikeCount', 0)
    title = j['snippet']['title']
    if 'localized' in j['snippet']:
        title = j['snippet']['localized'].get('title') or title

    out = ('\x02{title}\x02 - length \x02{duration}\x02 - '
           '{likes}\u2191{dislikes}\u2193 - '
           '\x02{views}\x02 views - '
           '\x02{snippet[channelTitle]}\x02 on \x02{published}\x02'
          ).format(duration=duration, likes=likes, dislikes=dislikes, views=views, published=published, title=title, **j)

    # TODO: figure out how to detect NSFW videos

    if 'rating' in j:
        out += ' - rated \x02%.2f/5.0\x02 (%d)' % (j['rating'],
                                                   j['ratingCount'])

    if 'viewCount' in j:
        out += ' - \x02%s\x02 views' % group_int_digits(j['viewCount'])

    upload_time_since = timesince.timesince(
        timegm(
            time.strptime(j['uploaded'], "%Y-%m-%dT%H:%M:%S.000Z")
        )
    )

    upload_time = time.strptime(j['uploaded'], "%Y-%m-%dT%H:%M:%S.000Z")

    out += ' - \x02%s\x02 ago by \x02%s\x02' % (upload_time_since, j['uploader'])

    if 'contentRating' in j:
        out += ' - \x034NSFW\x02'

    return out


def group_int_digits(number, delimiter=' ', grouping=3):
    base = str(number).strip()
    builder = []
    while base:
        builder.append(base[-grouping:])
        base = base[:-grouping]
    builder.reverse()
    return delimiter.join(builder)


@hook.api_key('google')
@hook.regex(*youtube_re)
def youtube_url(match, api_key=None):
    return get_video_description(match.group(1), api_key)


@hook.api_key('google')<commit_msg>use published as 'since' time instead of uploaded at<commit_after>from __future__ import unicode_literals

from builtins import str
import re
import time
from calendar import timegm
from util import hook, http, timesince


youtube_re = (r'(?:youtube.*?(?:v=|/v/)|youtu\.be/|yooouuutuuube.*?id=)'
              '([-_a-z0-9]+)', re.I)

BASE_URL = 'https://www.googleapis.com/youtube/v3/'
INFO_URL = BASE_URL + 'videos?part=snippet,contentDetails,statistics&hl=en'
SEARCH_API_URL = BASE_URL + 'search'
VIDEO_URL = 'https://youtu.be/%s'


def get_video_description(vid_id, api_key):
    j = http.get_json(INFO_URL, id=vid_id, key=api_key)

    if not j['pageInfo']['totalResults']:
        return

    j = j['items'][0]

    duration = j['contentDetails']['duration'].replace('PT', '').lower()

    published = j['snippet']['publishedAt'].replace('.000Z', 'Z')
    published = time.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
    published_since = timesince.timesince(timegm(published))
    published = time.strftime("%Y.%m.%d", published)

    views = group_int_digits(j['statistics']['viewCount'], ',')
    likes = j['statistics'].get('likeCount', 0)
    dislikes = j['statistics'].get('dislikeCount', 0)
    title = j['snippet']['title']
    if 'localized' in j['snippet']:
        title = j['snippet']['localized'].get('title') or title

    out = ('\x02{title}\x02 - length \x02{duration}\x02 - '
           '{likes}\u2191{dislikes}\u2193 - '
           '\x02{views}\x02 views - '
           '\x02{snippet[channelTitle]}\x02 on \x02{published}\x02'
          ).format(duration=duration, likes=likes, dislikes=dislikes, views=views, published=published, title=title, **j)

    # TODO: figure out how to detect NSFW videos

    if 'rating' in j:
        out += ' - rated \x02%.2f/5.0\x02 (%d)' % (j['rating'],
                                                   j['ratingCount'])

    if 'viewCount' in j:
        out += ' - \x02%s\x02 views' % group_int_digits(j['viewCount'])

    out += ' - \x02%s\x02 ago by \x02%s\x02' % (published_since, j['uploader'])

    if 'contentRating' in j:
        out += ' - \x034NSFW\x02'

    return out


def group_int_digits(number, delimiter=' ', grouping=3):
    base = str(number).strip()
    builder = []
    while base:
        builder.append(base[-grouping:])
        base = base[:-grouping]
    builder.reverse()
    return delimiter.join(builder)


@hook.api_key('google')
@hook.regex(*youtube_re)
def youtube_url(match, api_key=None):
    return get_video_description(match.group(1), api_key)


@hook.api_key('google')
{'fraction_non_alphanumeric': 0.18069070788017555, 'fraction_numerical': 0.025186033199771037, 'mean_word_length': 3.2548701298701297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803466', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1842, 'n_words': 339}