<commit_before>#!/usr/bin/env python

__all__ = ['tumblr_download']

from ..common import *
from .universal import *
from .dailymotion import dailymotion_download
from .vimeo import vimeo_download
from .vine import vine_download

def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
        universal_download(url, output_dir, merge=merge, info_only=info_only)
        return

    import ssl
    ssl_context = request.HTTPSHandler(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
    cookie_handler = request.HTTPCookieProcessor()
    opener = request.build_opener(ssl_context, cookie_handler)
    request.install_opener(opener)

    page = get_html(url)
    form_key = match1(page, r'id="tumblr_form_key" content="([^"]+)"')
    if form_key is not None:
        # bypass GDPR consent page
        referer = 'https://www.tumblr.com/privacy/consent?redirect=%s' % parse.quote_plus(url)
        post_content('https://www.tumblr.com/svc/privacy/consent',
                     headers={
                         'Content-Type': 'application/json',
                         'User-Agent': fake_headers['User-Agent'],
                         'Referer': referer,
                         'X-tumblr-form-key': form_key,
                         'X-Requested-With': 'XMLHttpRequest'
                     },
                     post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url)
        page = get_html(url, faker=True)

    html = parse.unquote(page).replace('\/', '/')
    feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)

    if feed in ['photo', 'photoset', 'entry'] or feed is None:
        # try to extract photos
        page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \
                     r1(r'<meta property="og:description" content="([^"\n]+)', html) or \
                     r1(r'<title>([^<\n]*)', html)
        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.jpg)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.png)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^";&]+_\d+\.gif)', html)

        tuggles = {}
        for url in urls:
            filename = parse.unquote(url.split('/')[-1])
            title = '.'.join(filename.split('.')[:-1])
            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
            quality = int(r1(r'^tumblr_.+_(\d+)$', title))
            ext = filename.split('.')[-1]
            try:
                size = int(get_head(url)['Content-Length'])
                if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
                    tuggles[tumblr_id] = {
                        'title': title,
                        'url': url,
                        'quality': quality,<commit_msg>[tumblr] always download the high res (1280) version of images<commit_after>#!/usr/bin/env python

__all__ = ['tumblr_download']

from ..common import *
from .universal import *
from .dailymotion import dailymotion_download
from .vimeo import vimeo_download
from .vine import vine_download

def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
        universal_download(url, output_dir, merge=merge, info_only=info_only)
        return

    import ssl
    ssl_context = request.HTTPSHandler(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
    cookie_handler = request.HTTPCookieProcessor()
    opener = request.build_opener(ssl_context, cookie_handler)
    request.install_opener(opener)

    page = get_html(url)
    form_key = match1(page, r'id="tumblr_form_key" content="([^"]+)"')
    if form_key is not None:
        # bypass GDPR consent page
        referer = 'https://www.tumblr.com/privacy/consent?redirect=%s' % parse.quote_plus(url)
        post_content('https://www.tumblr.com/svc/privacy/consent',
                     headers={
                         'Content-Type': 'application/json',
                         'User-Agent': fake_headers['User-Agent'],
                         'Referer': referer,
                         'X-tumblr-form-key': form_key,
                         'X-Requested-With': 'XMLHttpRequest'
                     },
                     post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url)
        page = get_html(url, faker=True)

    html = parse.unquote(page).replace('\/', '/')
    feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)

    if feed in ['photo', 'photoset', 'entry'] or feed is None:
        # try to extract photos
        page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \
                     r1(r'<meta property="og:description" content="([^"\n]+)', html) or \
                     r1(r'<title>([^<\n]*)', html)
        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.jpg)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.png)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^";&]+_\d+\.gif)', html)

        tuggles = {}
        for url in urls:
            hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg'  # FIXME: decide actual quality
            filename = parse.unquote(hd_url.split('/')[-1])
            title = '.'.join(filename.split('.')[:-1])
            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
            quality = int(r1(r'^tumblr_.+_(\d+)$', title))
            ext = filename.split('.')[-1]
            try:
                size = int(get_head(hd_url)['Content-Length'])
                if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
                    tuggles[tumblr_id] = {
                        'title': title,
                        'url': hd_url,
                        'quality': quality,
{'fraction_non_alphanumeric': 0.17602695763799744, 'fraction_numerical': 0.004974326059050065, 'mean_word_length': 2.4078731547293604, 'pattern_counts': {'":': 16, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6697667', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2022, 'n_words': 330}