<commit_before># """
# Test some simple harvesting scenarios
# """

# import os

# import pytest

# from ckan_api_client.syncing import CkanDataImportClient
# from .utils.harvest_source import HarvestSource


# HERE = os.path.abspath(os.path.dirname(__file__))
# DATA_DIR = os.path.join(os.path.dirname(HERE), 'data', 'random')

# HARVEST_SOURCE_NAME = 'dummy-harvest-source'


# @pytest.fixture(params=['day-{0:02d}'.format(x) for x in xrange(4)])
# def harvest_source(request):
#     return HarvestSource(DATA_DIR, request.param)<commit_msg>Some tests for syncing client<commit_after>"""
Test some simple harvesting scenarios, generating data on-the-fly
"""

# import os
import random

# import pytest

# from ckan_api_client.syncing import CkanDataImportClient
# from .utils.harvest_source import HarvestSource
from ckan_api_client.tests.utils.generate import (
    generate_organization, generate_group, generate_dataset)


def generate_data():
    organizations = {}
    groups = {}
    datasets = {}

    for i in xrange(3):
        org = generate_organization()
        org['id'] = org['name'] = 'org-{0}'.format(i)
        organizations[org['name']] = org

    for i in xrange(10):
        grp = generate_group()
        grp['id'] = grp['name'] = 'group-{0}'.format(i)
        groups[grp['id']] = grp

    for i in xrange(20):
        dataset = generate_dataset()
        dataset['id'] = 'dataset-{0}'.format(i)
        dataset['groups'] = random.sample(list(groups), random.randint(0, 3))
        dataset['owner_org'] = random.choice(list(organizations))
        datasets[dataset['id']] = dataset

    return {
        'organization': organizations,
        'group': groups,
        'dataset': datasets,
    }


def test_simple_harvesting(ckan_client_sync):
    client = ckan_client_sync
    data = generate_data()

    client.sync('dummy-source', data)
{'fraction_non_alphanumeric': 0.1385606874328679, 'fraction_numerical': 0.007518796992481203, 'mean_word_length': 3.752551020408163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '142670', 'n_tokens_mistral': 614, 'n_tokens_neox': 598, 'n_words': 126}