<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import uuid
import logging

logger = logging.getLogger(__name__)


# Module API

class Pipeline(object):
    """Pipeline representation (e.g. warehouse to database).

    Args:
        source (object): datastore connection
        target (object): datastore connection

    """

    # Public

    def __init__(self, source, target):
        self.__source = source
        self.__target = target

    def read(self, table, bufsize=10, orderby='meta_id'):
        """Read data from source.

        Yields:
            dict: the next item from table

        """
        offset = 0
        while True:
            query = {'_offset': offset, '_limit': bufsize, 'order_by': orderby}
            count = self.__source[table].find(return_count=True, **query)
            if not count:
                break
            items = self.__source[table].find(**query)
            offset += bufsize
            for item in items:
                item['meta_id'] = uuid.UUID(item['meta_id']).hex
                yield item

    def begin(self):<commit_msg>increased pipeline read buffer size<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import uuid
import logging

logger = logging.getLogger(__name__)


# Module API

class Pipeline(object):
    """Pipeline representation (e.g. warehouse to database).

    Args:
        source (object): datastore connection
        target (object): datastore connection

    """

    # Public

    def __init__(self, source, target):
        self.__source = source
        self.__target = target

    def read(self, table, bufsize=100, orderby='meta_id'):
        """Read data from source.

        Yields:
            dict: the next item from table

        """
        offset = 0
        while True:
            query = {'_offset': offset, '_limit': bufsize, 'order_by': orderby}
            count = self.__source[table].find(return_count=True, **query)
            if not count:
                break
            items = self.__source[table].find(**query)
            offset += bufsize
            for item in items:
                item['meta_id'] = uuid.UUID(item['meta_id']).hex
                yield item

    def begin(self):
{'fraction_non_alphanumeric': 0.09891435464414958, 'fraction_numerical': 0.0036188178528347406, 'mean_word_length': 2.269382391590013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1024503', 'n_tokens_mistral': 686, 'n_tokens_neox': 657, 'n_words': 205}