<commit_before>#!/usr/bin/env python

import requests
import os
from io import BytesIO

TRAIN_LIMIT = 100
VAL_LIMIT = 50
TRAIN = 1
VAL = 2

BRAIN_ROOT = os.environ.get('BRAIN_ROOT', 'http://127.0.0.1:3000/api')

def get_pins(uri):
    api = 'http://api.huaban.com/{}'.format(uri)
    url = api

    while True:
        rsp = requests.get(url, headers={
            'Accept': 'application/json',
            'User-Agent': 'Huabot/1.0'})
        pins = rsp.json()['pins']
        if len(pins) == 0:
            break

        min_pin_id = pins[0]['pin_id']
        for pin in pins:
            if min_pin_id > pin['pin_id']:
                min_pin_id = pin['pin_id']

            yield pin

        url = api + '?limit=100&max=%s'%min_pin_id


def submit_pin(pin, tag, data_type=1):
    file_url = 'http://img.hb.aicdn.com/' + pin['file']['key'] + '_fw320'
    print('Submit dataset: %s (%s)'%(tag, file_url))
    try:
        res = requests.get(file_url, timeout=30)
        data = res.content
        f = BytesIO()
        f.write(data)
        f.seek(0, 0)<commit_msg>Fix. No return data crash<commit_after>#!/usr/bin/env python

import requests
import os
from io import BytesIO

TRAIN_LIMIT = 100
VAL_LIMIT = 50
TRAIN = 1
VAL = 2

BRAIN_ROOT = os.environ.get('BRAIN_ROOT', 'http://127.0.0.1:3000/api')

def get_pins(uri):
    api = 'http://api.huaban.com/{}'.format(uri)
    url = api

    while True:
        rsp = requests.get(url, headers={
            'Accept': 'application/json',
            'User-Agent': 'Huabot/1.0'})
        pins = rsp.json().get('pins', [])
        if len(pins) == 0:
            break

        min_pin_id = pins[0]['pin_id']
        for pin in pins:
            if min_pin_id > pin['pin_id']:
                min_pin_id = pin['pin_id']

            yield pin

        url = api + '?limit=100&max=%s'%min_pin_id


def submit_pin(pin, tag, data_type=1):
    file_url = 'http://img.hb.aicdn.com/' + pin['file']['key'] + '_fw320'
    print('Submit dataset: %s (%s)'%(tag, file_url))
    try:
        res = requests.get(file_url, timeout=30)
        data = res.content
        f = BytesIO()
        f.write(data)
        f.seek(0, 0)
{'fraction_non_alphanumeric': 0.16342592592592592, 'fraction_numerical': 0.02962962962962963, 'mean_word_length': 2.206231454005935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3286023', 'n_tokens_mistral': 861, 'n_tokens_neox': 793, 'n_words': 167}