<commit_before>import sys
import copy
import requests

from .zdesk_api import ZendeskAPI

if sys.version < '3':
    from httplib import responses
    from urlparse import urlsplit
else:
    from http.client import responses
    from urllib.parse import urlsplit


def get_id_from_url(url):
    url_id = urlsplit(url).path.split('/')[-1].split('.')[0]
    if url_id.isdigit():
        return url_id
    else:
        return None<commit_msg>add batch helper<commit_after>import sys
import copy
import requests

from .zdesk_api import ZendeskAPI

if sys.version < '3':
    from httplib import responses
    from urlparse import urlsplit
else:
    from http.client import responses
    from urllib.parse import urlsplit


def batch(sequence, callback, size=100, **kwargs):
    """Helper to setup batch requests.

    There are endpoints which support updating multiple resources at once,
    but they are often limited to 100 updates per request.
    This function helps with splitting bigger requests into sequence of
    smaller ones.

    Example:
        def add_organization_tag(organizations, tag):
            request = {'organizations': [
                {
                    'id': org['id'],
                    'tags': org['tags'] + [tag],
                } for org in organizations
            ]}
            job = z.organizations_update_many(request)['job_status']
            return job['id']

        # z = Zendesk(...)
        orgs = z.organizations_list(get_all_pages=True)['organizations']
        job_ids = [job for job in
                   batch(orgs, add_organization_tag, tag='new_tag')]

    Parameters:
        sequence - any sequence you want to split
        callback - function to call with slices of sequence,
            its return value is yielded on each slice
        size - size of chunks, combined with length of sequence determines
            how many times callback is called (defaults to 100)
        **kwargs - any additional keyword arguments are passed to callback
    """
    batch_len, rem = divmod(len(sequence), size)
    if rem > 0:
        batch_len += 1
    for i in range(batch_len):
        offset = i * size
        yield callback(sequence[offset:offset + size], **kwargs)


def get_id_from_url(url):
    url_id = urlsplit(url).path.split('/')[-1].split('.')[0]
    if url_id.isdigit():
        return url_id
    else:
        return None
{'fraction_non_alphanumeric': 0.0875943000838223, 'fraction_numerical': 0.007124895222129086, 'mean_word_length': 2.4846715328467153, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985751', 'n_tokens_mistral': 679, 'n_tokens_neox': 654, 'n_words': 228}