<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
from .. import base
logger = logging.getLogger(__name__)


# Module API

def process(conf, conn):
    """Remove records without trial.
    """

    count = 0
    for trial in base.helpers.iter_rows(conn, 'database', 'trials', orderby='id'):

        # Get all records
        records = list(conn['database']['records'].find(trial_id=trial['id']))

        # Trial has no multiple records
        if len(records) <= 1:
            continue

        # Prepare identifier segments
        segments = []
        for record in records:

            # Get set of record identifiers
            idset = set(record['identifiers'].items())

            # Find intersections -> update segments
            intersection = False
            for segment in segments:
                if segment.intersection(idset):
                    segment.update(idset)
                    intersection = True

            # No intersection -> new segment
            if not intersection:
                segments.append(idset)

        # Sort segments and get the biggest segment
        segments = list(sorted(segments, key=lambda s: len(s), reverse=True))
        biggest_segment = segments[0]

        # Delete all records without intersection with the biggest segment
        for record in records:
            idset = set(record['identifiers'].items())
            if not biggest_segment.intersection(idset):
                if conn['database']['records'].find_one(id=record['id']):
                    conn['database']['records'].delete(id=record['id'])
                    logger.info('Removed record: %s', record['identifiers'])

        # Log info
        count += 1
        if count and not count % 100:
            logger.info('Processed %s trials', count)<commit_msg>[#47] fixed counter in record_remover<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
from .. import base
logger = logging.getLogger(__name__)


# Module API

def process(conf, conn):
    """Remove records without trial.
    """

    count = 0
    for trial in base.helpers.iter_rows(conn, 'database', 'trials', orderby='id'):

        # Count trials
        count += 1

        # Get all records
        records = list(conn['database']['records'].find(trial_id=trial['id']))

        # Trial has no multiple records
        if len(records) <= 1:
            continue

        # Prepare identifier segments
        segments = []
        for record in records:

            # Get set of record identifiers
            idset = set(record['identifiers'].items())

            # Find intersections -> update segments
            intersection = False
            for segment in segments:
                if segment.intersection(idset):
                    segment.update(idset)
                    intersection = True

            # No intersection -> new segment
            if not intersection:
                segments.append(idset)

        # Sort segments and get the biggest segment
        segments = list(sorted(segments, key=lambda s: len(s), reverse=True))
        biggest_segment = segments[0]

        # Delete all records without intersection with the biggest segment
        for record in records:
            idset = set(record['identifiers'].items())
            if not biggest_segment.intersection(idset):
                if conn['database']['records'].find_one(id=record['id']):
                    conn['database']['records'].delete(id=record['id'])
                    logger.info('Removed record: %s', record['identifiers'])

        # Log info
        if count and not count % 100:
            logger.info('Processed %s trials', count)
{'fraction_non_alphanumeric': 0.10040363269424823, 'fraction_numerical': 0.004540867810292634, 'mean_word_length': 2.1898632341110216, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673958', 'n_tokens_mistral': 1008, 'n_tokens_neox': 957, 'n_words': 305}