<commit_before>#!/usr/bin/env python

__all__ = ['twitter_download']

from ..common import *

def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    html = get_html(url)
    screen_name = r1(r'data-screen-name="([^"]*)"', html)
    item_id = r1(r'data-item-id="([^"]*)"', html)
    page_title = "{} [{}]".format(screen_name, item_id)

    icards = r1(r'data-src="([^"]*)"', html)
    if icards:
        html = get_html("https://twitter.com" + icards)
        data = json.loads(unescape_html(r1(r'data-player-config="([^"]*)"', html)))
        source = data['playlist'][0]['source']<commit_msg>[twitter] support embedded Vine<commit_after>#!/usr/bin/env python

__all__ = ['twitter_download']

from ..common import *
from .vine import vine_download

def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    html = get_html(url)
    screen_name = r1(r'data-screen-name="([^"]*)"', html)
    item_id = r1(r'data-item-id="([^"]*)"', html)
    page_title = "{} [{}]".format(screen_name, item_id)

    icards = r1(r'data-src="([^"]*)"', html)
    if icards:
        html = get_html("https://twitter.com" + icards)
        data_player_config = r1(r'data-player-config="([^"]*)"', html)
        if data_player_config is None:
            vine_src = r1(r'<iframe src="([^"]*)"', html)
            vine_download(vine_src, output_dir=output_dir, merge=merge, info_only=info_only)
            return
        data = json.loads(unescape_html(data_player_config))
        source = data['playlist'][0]['source']
{'fraction_non_alphanumeric': 0.2057877813504823, 'fraction_numerical': 0.00707395498392283, 'mean_word_length': 3.9554140127388533, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6867628', 'n_tokens_mistral': 573, 'n_tokens_neox': 565, 'n_words': 95}