<commit_before>#!/usr/bin/env python2
# vim: set fileencoding=utf8

import os
import sys
import re
import json
from collections import deque
import requests
import argparse
import random
import subprocess
import time

api_key = '<KEY>'

############################################################
# wget exit status
wget_es = {
    0: "No problems occurred.",
    2: "User interference.",
    1<<8: "Generic error code.",
    2<<8: "Parse error - for instance, when parsing command-line " \
        "optio.wgetrc or .netrc...",
    3<<8: "File I/O error.",
    4<<8: "Network failure.",
    5<<8: "SSL verification failure.",
    6<<8: "Username/password authentication failure.",
    7<<8: "Protocol errors.",
    8<<8: "Server issued an error response."
}
############################################################

s = '\x1b[%d;%dm%s\x1b[0m'       # terminual color template

headers = {
    "Accept":"text/html,application/xhtml+xml,application/xml; " \
        "q=0.9,image/webp,*/*;q=0.8",
    "Accept-Encoding":"text/html",
    "Accept-Language":"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2",
    "Content-Type":"application/x-www-form-urlencoded",
    "Referer":"https://api.tumblr.com/console//calls/blog/posts",
    "User-Agent":"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 " \
        "(KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
}

ss = requests.session()
ss.headers.update(headers)

def check_queue(queue, cb):
    for f in queue:
        st = f[0].poll()
        if st is not None:
            if st == 0: cb(f[1])
            queue.remove(f)
    return queue

def sleep(size, num):
    t = float(size) / num
    time.sleep(t)

def async(tasks, queue, run=None, cb=None, num=10):
    queue = check_queue(queue, cb)
    sleep(len(queue), num)
    nsize = num - len(queue)
    for i in xrange(nsize):
        try:
            task = tasks.popleft()
        except IndexError:
            break
        f = run(task)
        if f: queue.append(f)
    #return tasks, queue

class tumblr(object):
    def __init__(self):
        self.queue = []
        self.tasks = deque()
        self.key = 0

    def save_json(self):
        with open(self.json_path, 'w') as g:
            g.write(json.dumps(
                {'key': self.key}, indent=4, sort_keys=True))

    def get_site_infos(self, postid=None):
        self.infos['photos'] = []
        self.url = 'https://api.tumblr.com/v2/blog/%s/posts/photo' \
            % self.infos['host']
        params = {
            "offset": self.key if not postid else "",
            "limit": 20 if not postid else "",
            "type": "photo",<commit_msg>[tumblr] restructure code<commit_after>#!/usr/bin/env python2
# vim: set fileencoding=utf8

import os
import sys
import re
import json
from collections import deque
import requests
import argparse
import random
import subprocess
import time

api_key = '<KEY>'

############################################################
# wget exit status
wget_es = {
    0: "No problems occurred.",
    2: "User interference.",
    1<<8: "Generic error code.",
    2<<8: "Parse error - for instance, when parsing command-line " \
        "optio.wgetrc or .netrc...",
    3<<8: "File I/O error.",
    4<<8: "Network failure.",
    5<<8: "SSL verification failure.",
    6<<8: "Username/password authentication failure.",
    7<<8: "Protocol errors.",
    8<<8: "Server issued an error response."
}
############################################################

s = '\x1b[%d;%dm%s\x1b[0m'       # terminual color template

headers = {
    "Accept":"text/html,application/xhtml+xml,application/xml; " \
        "q=0.9,image/webp,*/*;q=0.8",
    "Accept-Encoding":"text/html",
    "Accept-Language":"en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2",
    "Content-Type":"application/x-www-form-urlencoded",
    "Referer":"https://api.tumblr.com/console//calls/blog/posts",
    "User-Agent":"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 " \
        "(KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
}

ss = requests.session()
ss.headers.update(headers)

def check_queue(queue, cb):
    for f in queue:
        st = f[0].poll()
        if st is not None:
            if st == 0: cb(f[1])
            queue.remove(f)
    return queue

def sleep(size, num):
    t = float(size) / num
    time.sleep(t)

def async(tasks, queue, run=None, cb=None, num=10):
    queue = check_queue(queue, cb)
    sleep(len(queue), num)
    nsize = num - len(queue)
    for i in xrange(nsize):
        try:
            task = tasks.popleft()
        except IndexError:
            break
        f = run(task)
        if f: queue.append(f)
    #return tasks, queue

class TumblrAPI(object):
    def _request(self, base_hostname, target, params, type=''):

    def _info(self, base_hostname):

    def _followers(self, base_hostname):

    def _likes(self, base_hostname):

    def _photo(self, base_hostname, offset='', tag='', id=''):

    def _audio(self, base_hostname, offset='', tag='', id=''):

    def _video(self, base_hostname, offset='', tag='', id=''):

class Tumblr(TumblrAPI):
    def __init__(self):
        self.queue = []
        self.tasks = deque()
        self.key = 0

    def save_json(self):
        with open(self.json_path, 'w') as g:
            g.write(json.dumps(
                {'key': self.key}, indent=4, sort_keys=True))

    def get_site_infos(self, postid=None):
        self.infos['photos'] = []
        self.url = 'https://api.tumblr.com/v2/blog/%s/posts/photo' \
            % self.infos['host']
        params = {
            "offset": self.key if not postid else "",
            "limit": 20 if not postid else "",
            "type": "photo",
{'fraction_non_alphanumeric': 0.19813314547375838, 'fraction_numerical': 0.02500880591757661, 'mean_word_length': 2.704500978473581, 'pattern_counts': {'":': 18, '<': 37, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2293190', 'n_tokens_mistral': 1997, 'n_tokens_neox': 1922, 'n_words': 450}