<commit_before># encoding:utf-8

from urllib.parse import urlparse
import requests
import yaml
import os
import imghdr
import time

from imgurpython import ImgurClient


telegram_autoplay_limit = 10 * 1024 * 1024


def get_url(submission):
    def what_is_inside(url):
        header = requests.head(url).headers
        if 'Content-Type' in header:
            return header['Content-Type']
        else:
            return ''

    url = submission.url
    print(submission.short_link)

    url_content = what_is_inside(url)
    print(url_content)
    if ('image/jpeg' == url_content or 'image/png' == url_content):
        return 'img', url, url_content.split('/')[1]

    if 'image/gif' in url_content:
        return 'gif', url, 'gif'
    
    if url.endswith('.gifv'):
        if 'image/gif' in what_is_inside(url[0:-1]):
            return 'gif', url[0:-1], 'gif'

    if submission.is_self is True:
        # Self submission with text
        return 'text', None, None

    if urlparse(url).netloc == 'imgur.com':
        # Imgur
        imgur_config = yaml.load(open('imgur.yml').read())
        imgur_client = ImgurClient(imgur_config['client_id'], imgur_config['client_secret'])
        path_parts = urlparse(url).path.split('/')
        if path_parts[1] == 'gallery':
            # TODO: gallary handling
            return 'other', url, None
        elif path_parts[1] == 'a':
            # An imgur album
            album = imgur_client.get_album(path_parts[2])
            story = {}
            for num, img in enumerate(album.images):
                number = num + 1
                story[number] = {
                    'link': img['link'],
                    'gif': img['animated'],
                    'type': img['type'].split('/')[1]
                }
            return 'album', story, None
        else:
            # Not an imgur album
            img = imgur_client.get_image(path_parts[1].split('.')[0])
            if not img.animated:
                return 'img', img.link, img.type.split('/')[1]
            else:
                return 'gif', img.link, 'gif'
            

    else:
        return 'other', url, None


def download_file(url, filename):
    # http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    chunk_counter = 0
    with open(filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks<commit_msg>Placeholder for imgur topics processing.<commit_after># encoding:utf-8

from urllib.parse import urlparse
import requests
import yaml
import os
import imghdr
import time

from imgurpython import ImgurClient


telegram_autoplay_limit = 10 * 1024 * 1024


def get_url(submission):
    def what_is_inside(url):
        header = requests.head(url).headers
        if 'Content-Type' in header:
            return header['Content-Type']
        else:
            return ''

    url = submission.url

    url_content = what_is_inside(url)
    if ('image/jpeg' == url_content or 'image/png' == url_content):
        return 'img', url, url_content.split('/')[1]

    if 'image/gif' in url_content:
        return 'gif', url, 'gif'
    
    if url.endswith('.gifv'):
        if 'image/gif' in what_is_inside(url[0:-1]):
            return 'gif', url[0:-1], 'gif'

    if submission.is_self is True:
        # Self submission with text
        return 'text', None, None

    if urlparse(url).netloc == 'imgur.com':
        # Imgur
        imgur_config = yaml.load(open('imgur.yml').read())
        imgur_client = ImgurClient(imgur_config['client_id'], imgur_config['client_secret'])
        path_parts = urlparse(url).path.split('/')
        if path_parts[1] == 'gallery':
            # TODO: gallary handling
            return 'other', url, None
        elif path_parts[1] == 'topic':
            # TODO: topic handling
            return 'other', url, None
        elif path_parts[1] == 'a':
            # An imgur album
            album = imgur_client.get_album(path_parts[2])
            story = {}
            for num, img in enumerate(album.images):
                number = num + 1
                story[number] = {
                    'link': img['link'],
                    'gif': img['animated'],
                    'type': img['type'].split('/')[1]
                }
            return 'album', story, None
        else:
            # Just imgur img
            img = imgur_client.get_image(path_parts[1].split('.')[0])
            if not img.animated:
                return 'img', img.link, img.type.split('/')[1]
            else:
                return 'gif', img.link, 'gif'
            

    else:
        return 'other', url, None


def download_file(url, filename):
    # http://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    chunk_counter = 0
    with open(filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
{'fraction_non_alphanumeric': 0.12618357487922705, 'fraction_numerical': 0.014492753623188406, 'mean_word_length': 2.0482921083627796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '690653', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1537, 'n_words': 382}