<commit_before>from itertools import islice, chain
from luminoso_api.v4_client import LuminosoClient
from luminoso_api.v4_constants import URL_BASE
from luminoso_api.v4_json_stream import transcode_to_stream, stream_json_lines


# http://code.activestate.com/recipes/303279-getting-items-in-batches/
def batches(iterable, size):
    """
    Take an iterator and yield its contents in groups of `size` items.
    """
    sourceiter = iter(iterable)
    while True:
        batchiter = islice(sourceiter, size)
        yield chain([next(batchiter)], batchiter)


def upload_stream(stream, server, account, projname, language=None,
                  username=None, password=<PASSWORD>,
                  append=False, stage=False):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = LuminosoClient.connect(server,
                                    username=username, password=password)
    if not append:
        # If we're not appending to an existing project, create new project.
        info = client.post('/projects/' + account, name=projname)
        project_id = info['project_id']
        print('New project ID:', project_id)
    else:
        projects = client.get('/projects/' + account, name=projname)
        if len(projects) == 0:
            print('No such project exists!')
            return
        if len(projects) > 1:
            print('Warning: Multiple projects with name "%s".  ' % projname,
                  end='')
        project_id = projects[0]['project_id']
        print('Using existing project with id %s.' % project_id)

    project = client.change_path('/projects/' + account + '/' + project_id)

    counter = 0
    for batch in batches(stream, 1000):
        counter += 1
        documents = list(batch)
        project.upload('docs', documents)
        print('Uploaded batch #%d' % (counter))

    if not stage:
        # Calculate the docs into the assoc space.
        print('Calculating.')<commit_msg>Fix bug in v4_upload import<commit_after>from itertools import islice, chain
from luminoso_api.v4_client import v4LuminosoClient
from luminoso_api.v4_constants import URL_BASE
from luminoso_api.v4_json_stream import transcode_to_stream, stream_json_lines


# http://code.activestate.com/recipes/303279-getting-items-in-batches/
def batches(iterable, size):
    """
    Take an iterator and yield its contents in groups of `size` items.
    """
    sourceiter = iter(iterable)
    while True:
        batchiter = islice(sourceiter, size)
        yield chain([next(batchiter)], batchiter)


def upload_stream(stream, server, account, projname, language=None,
                  username=None, password=<PASSWORD>,
                  append=False, stage=False):
    """
    Given a file-like object containing a JSON stream, upload it to
    Luminoso with the given account name and project name.
    """
    client = v4LuminosoClient.connect(server,
                                      username=username, password=password)
    if not append:
        # If we're not appending to an existing project, create new project.
        info = client.post('/projects/' + account, name=projname)
        project_id = info['project_id']
        print('New project ID:', project_id)
    else:
        projects = client.get('/projects/' + account, name=projname)
        if len(projects) == 0:
            print('No such project exists!')
            return
        if len(projects) > 1:
            print('Warning: Multiple projects with name "%s".  ' % projname,
                  end='')
        project_id = projects[0]['project_id']
        print('Using existing project with id %s.' % project_id)

    project = client.change_path('/projects/' + account + '/' + project_id)

    counter = 0
    for batch in batches(stream, 1000):
        counter += 1
        documents = list(batch)
        project.upload('docs', documents)
        print('Uploaded batch #%d' % (counter))

    if not stage:
        # Calculate the docs into the assoc space.
        print('Calculating.')
{'fraction_non_alphanumeric': 0.10321324245374879, 'fraction_numerical': 0.00949367088607595, 'mean_word_length': 2.6202643171806166, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2941295', 'n_tokens_mistral': 1200, 'n_tokens_neox': 1134, 'n_words': 337}