<commit_before># -*- coding: utf-8 -*-
""" Importing event data from a package """

import logging
import requests
from datetime import datetime as dt
from frictionless import Package, Resource
from .user.models import Event, Project, Activity, Category, User, Role
from .utils import format_date
from .apiutils import (
    get_project_list,
    get_event_users,
    get_event_activities,
    get_event_categories,
)


def PackageEvent(event, author=None, host_url='', full_contents=False):
    """ Creates a Data Package from the data of an event """

    # Define the author, if available
    contributors = []
    if author and not author.is_anonymous:
        contributors.append({
            "title": author.username,
            "path": author.webpage_url or '',
            "role": "author"
        })
    else:
        # Disallow anon access to full data
        full_contents = False

    # Set up a data package object
    package = Package(
        name='event-%d' % event.id,
        title=event.name,
        description="Event and project details collected with dribdat",
        keywords=["dribdat", "hackathon", "co-creation"],
        sources=[{"title": "dribdat", "path": "https://dribdat.cc"}],
        licenses=[{
            "name": "ODC-PDDL-1.0",
            "path": "http://opendatacommons.org/licenses/pddl/",
            "title": "Open Data Commons Public Domain Dedication & License 1.0"
        }],
        contributors=contributors,
        homepage=event.webpage_url or '',
        created=format_date(dt.now(), '%Y-%m-%dT%H:%M'),
        version="0.1.0",
    )

    # if False:  # as CSV
    #     fp_projects = tempfile.NamedTemporaryFile(
    #         mode='w+t', prefix='projects-', suffix='.csv')
    #     # print("Writing to temp CSV file", fp_projects.name)
    #     fp_projects.write(gen_csv(get_project_list(event.id)))
    #     resource = Resource(fp_projects.name)
    # if False:
    #     # print("Generating in-memory rowset")
    #     project_rows = gen_rows(get_project_list(event.id))
    #     resource = Resource(
    #         name='projects',
    #         data=project_rows,
    #     )

    # Generate resources

    # print("Generating in-memory JSON of event")
    package.add_resource(Resource(
            name='events',
            data=[event.get_full_data()],
        ))
    # print("Generating in-memory JSON of projects")
    package.add_resource(Resource(
            name='projects',
            data=get_project_list(event.id, host_url, True),
        ))
    if full_contents:
        # print("Generating in-memory JSON of participants")
        package.add_resource(Resource(
                name='users',
                data=get_event_users(event, full_contents),
            ))
        # print("Generating in-memory JSON of activities")
        package.add_resource(Resource(
                name='activities',
                data=get_event_activities(event.id, 500),
            ))
        # print("Generating in-memory JSON of activities")
        package.add_resource(Resource(
                name='categories',
                data=get_event_categories(event.id),
            ))
        # print("Adding supplementary README")
        package.add_resource(Resource(
                name='readme',
                path='PACKAGE.txt',
            ))

    return package


def importEvents(data, DRY_RUN=False):
    updates = []
    for evt in data:
        name = evt['name']
        event = Event.query.filter_by(name=name).first()
        if not event:
            logging.info('Creating event: %s' % name)
            event = Event()
        else:
            logging.info('Updating event: %s' % name)
        event.set_from_data(evt)
        if not DRY_RUN:
            event.save()
        updates.append(event.data)
    return updates


def importCategories(data, DRY_RUN=False):
    updates = []
    for ctg in data:
        name = ctg['name']
        category = Category.query.filter_by(name=name).first()
        if not category:
            logging.info('Creating category: %s' % name)
            category = Category()
        else:
            logging.info('Updating category: %s' % name)
        category.set_from_data(ctg)
        if not DRY_RUN:
            category.save()
        updates.append(category.data)
    return updates


def importUsers(data, DRY_RUN=False):
    updates = []
    for usr in data:
        name = usr['username']
        user = User.query.filter_by(username=name).first()
        if user:
            # Do not update existing user data
            logging.info('Skipping user: %s' % name)
            continue
        logging.info('Creating user: %s' % name)
        user = User()
        user.set_from_data(usr)
        importUserRoles(user, usr['roles'], DRY_RUN)
        if not DRY_RUN:
            user.save()<commit_msg>Check user email on import<commit_after># -*- coding: utf-8 -*-
""" Importing event data from a package """

import logging
import requests
from datetime import datetime as dt
from frictionless import Package, Resource
from .user.models import Event, Project, Activity, Category, User, Role
from .utils import format_date
from .apiutils import (
    get_project_list,
    get_event_users,
    get_event_activities,
    get_event_categories,
)


def PackageEvent(event, author=None, host_url='', full_contents=False):
    """ Creates a Data Package from the data of an event """

    # Define the author, if available
    contributors = []
    if author and not author.is_anonymous:
        contributors.append({
            "title": author.username,
            "path": author.webpage_url or '',
            "role": "author"
        })
    else:
        # Disallow anon access to full data
        full_contents = False

    # Set up a data package object
    package = Package(
        name='event-%d' % event.id,
        title=event.name,
        description="Event and project details collected with dribdat",
        keywords=["dribdat", "hackathon", "co-creation"],
        sources=[{"title": "dribdat", "path": "https://dribdat.cc"}],
        licenses=[{
            "name": "ODC-PDDL-1.0",
            "path": "http://opendatacommons.org/licenses/pddl/",
            "title": "Open Data Commons Public Domain Dedication & License 1.0"
        }],
        contributors=contributors,
        homepage=event.webpage_url or '',
        created=format_date(dt.now(), '%Y-%m-%dT%H:%M'),
        version="0.1.0",
    )

    # if False:  # as CSV
    #     fp_projects = tempfile.NamedTemporaryFile(
    #         mode='w+t', prefix='projects-', suffix='.csv')
    #     # print("Writing to temp CSV file", fp_projects.name)
    #     fp_projects.write(gen_csv(get_project_list(event.id)))
    #     resource = Resource(fp_projects.name)
    # if False:
    #     # print("Generating in-memory rowset")
    #     project_rows = gen_rows(get_project_list(event.id))
    #     resource = Resource(
    #         name='projects',
    #         data=project_rows,
    #     )

    # Generate resources

    # print("Generating in-memory JSON of event")
    package.add_resource(Resource(
            name='events',
            data=[event.get_full_data()],
        ))
    # print("Generating in-memory JSON of projects")
    package.add_resource(Resource(
            name='projects',
            data=get_project_list(event.id, host_url, True),
        ))
    if full_contents:
        # print("Generating in-memory JSON of participants")
        package.add_resource(Resource(
                name='users',
                data=get_event_users(event, full_contents),
            ))
        # print("Generating in-memory JSON of activities")
        package.add_resource(Resource(
                name='activities',
                data=get_event_activities(event.id, 500),
            ))
        # print("Generating in-memory JSON of activities")
        package.add_resource(Resource(
                name='categories',
                data=get_event_categories(event.id),
            ))
        # print("Adding supplementary README")
        package.add_resource(Resource(
                name='readme',
                path='PACKAGE.txt',
            ))

    return package


def importEvents(data, DRY_RUN=False):
    updates = []
    for evt in data:
        name = evt['name']
        event = Event.query.filter_by(name=name).first()
        if not event:
            logging.info('Creating event: %s' % name)
            event = Event()
        else:
            logging.info('Updating event: %s' % name)
        event.set_from_data(evt)
        if not DRY_RUN:
            event.save()
        updates.append(event.data)
    return updates


def importCategories(data, DRY_RUN=False):
    updates = []
    for ctg in data:
        name = ctg['name']
        category = Category.query.filter_by(name=name).first()
        if not category:
            logging.info('Creating category: %s' % name)
            category = Category()
        else:
            logging.info('Updating category: %s' % name)
        category.set_from_data(ctg)
        if not DRY_RUN:
            category.save()
        updates.append(category.data)
    return updates


def importUsers(data, DRY_RUN=False):
    updates = []
    for usr in data:
        name = usr['username']
        email = usr['email']
        if User.query.filter_by(username=name).first() or \
           User.query.filter_by(email=email).first():
            # Do not update existing user data
            logging.info('Skipping user: %s' % name)
            continue
        logging.info('Creating user: %s' % name)
        user = User()
        user.set_from_data(usr)
        importUserRoles(user, usr['roles'], DRY_RUN)
        if not DRY_RUN:
            user.save()
{'fraction_non_alphanumeric': 0.11819484240687679, 'fraction_numerical': 0.0022513303315595578, 'mean_word_length': 2.159715486582606, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6775478', 'n_tokens_mistral': 2856, 'n_tokens_neox': 2807, 'n_words': 653}