<commit_before>import logging
import sys

from modularodm import Q
from modularodm.storage.base import KeyExistsException

from framework.mongo import database
from framework.transactions.context import TokuTransaction
from scripts import utils as script_utils
from website.app import init_app
from website import models 

logger = logging.getLogger(__name__)


def get_targets():
    return database.node.find({'preprint_file': {'$ne': None}})

def validate_node_document(document):
    logger.info('* Validating and repairing node {}'.format(document['_id']))    

    assert document.get('preprint_created'), '{} has no preprint_created'.format(document['_id'])
    assert document.get('preprint_file'), '{} has no preprint_file'.format(document['_id'])
    assert document.get('preprint_subjects'), '{} has no preprint_subjects'.format(document['_id'])

    if not document.get('preprint_providers'):
        logger.debug('{} has no preprint_providers, assuming OSF'.format(document['_id']))
        database['node'].find_and_modify(
            {'_id': document['_id']},
            {'$set': {
                'preprint_providers': ['osf']
            }}
        )

def validate_node_preprint_subjects(node):
    flat_subjects = node.preprint_subjects
    for subject_id in flat_subjects:
        subject = models.Subject.load(subject_id)
        if not subject:
            logger.debug('Found nonexistant subject {} on node {}, removing'.format(subject_id, node._id))
            node.preprint_subjects.remove(subject_id)
            node.save()
            validate_node_preprint_subjects(node)
            break
        if subject.parents and not set([c._id for c in subject.parents]) & set(flat_subjects):
            logger.debug('Found subject {} on node {} without parents. Adding first parent - {}'.format(subject_id, node._id, subject.parents[0]._id))
            node.preprint_subjects.append(subject.parents[0]._id)
            node.save()
            validate_node_preprint_subjects(node)
            break


def create_preprint_service_from_node(document):
    created = {}
    for provider_id in document['preprint_providers']:
        node = models.Node.load(document['_id'])
        provider = models.PreprintProvider.load(provider_id)
        # primary_file already set correctly* on node
        if not provider:
            logger.warn('Unable to find provider {} for node {}, skipping'.format(provider_id, document['_id']))
            continue

        try:
            logger.info('* Creating preprint for node {}'.format(node._id))
            preprint = models.PreprintService(node=node, provider=provider)
            preprint.save()
            database['preprintservice'].find_and_modify(
                {'_id': document['_id']},
                {'$set': {
                    'date_created': document['preprint_created']
                }}
            )
        except KeyExistsException:
            logger.warn('Duplicate PreprintService found for provider {} on node {}, skipping'.format(provider._id, node._id))
            continue
        else:
            if preprint.provider._id == 'osf':
                # Give Guid retention priotity to OSF-provider
                # Probably won't matter; shouldn't be non-osf
                swap_guids(node, preprint)
            node.reload()
            preprint.reload()<commit_msg>Set publish fields<commit_after>import logging
import sys

from modularodm import Q
from modularodm.storage.base import KeyExistsException

from framework.mongo import database
from framework.transactions.context import TokuTransaction
from scripts import utils as script_utils
from website.app import init_app
from website import models 

logger = logging.getLogger(__name__)


def get_targets():
    return database.node.find({'preprint_file': {'$ne': None}})

def validate_node_document(document):
    logger.info('* Validating and repairing node {}'.format(document['_id']))    

    assert document.get('preprint_created'), '{} has no preprint_created'.format(document['_id'])
    assert document.get('preprint_file'), '{} has no preprint_file'.format(document['_id'])
    assert document.get('preprint_subjects'), '{} has no preprint_subjects'.format(document['_id'])

    if not document.get('preprint_providers'):
        logger.debug('{} has no preprint_providers, assuming OSF'.format(document['_id']))
        database['node'].find_and_modify(
            {'_id': document['_id']},
            {'$set': {
                'preprint_providers': ['osf']
            }}
        )

def validate_node_preprint_subjects(node):
    flat_subjects = node.preprint_subjects
    for subject_id in flat_subjects:
        subject = models.Subject.load(subject_id)
        if not subject:
            logger.debug('Found nonexistant subject {} on node {}, removing'.format(subject_id, node._id))
            node.preprint_subjects.remove(subject_id)
            node.save()
            validate_node_preprint_subjects(node)
            break
        if subject.parents and not set([c._id for c in subject.parents]) & set(flat_subjects):
            logger.debug('Found subject {} on node {} without parents. Adding first parent - {}'.format(subject_id, node._id, subject.parents[0]._id))
            node.preprint_subjects.append(subject.parents[0]._id)
            node.save()
            validate_node_preprint_subjects(node)
            break


def create_preprint_service_from_node(document):
    created = {}
    for provider_id in document['preprint_providers']:
        node = models.Node.load(document['_id'])
        provider = models.PreprintProvider.load(provider_id)
        # primary_file already set correctly* on node
        if not provider:
            logger.warn('Unable to find provider {} for node {}, skipping'.format(provider_id, document['_id']))
            continue

        try:
            logger.info('* Creating preprint for node {}'.format(node._id))
            preprint = models.PreprintService(node=node, provider=provider)
            preprint.save()
            database['preprintservice'].find_and_modify(
                {'_id': document['_id']},
                {'$set': {
                    'date_created': document['preprint_created'],
                    'date_published': document['preprint_created'],
                    'is_published': True
                }}
            )
        except KeyExistsException:
            logger.warn('Duplicate PreprintService found for provider {} on node {}, skipping'.format(provider._id, node._id))
            continue
        else:
            if preprint.provider._id == 'osf':
                # Give Guid retention priotity to OSF-provider
                # Probably won't matter; shouldn't be non-osf
                swap_guids(node, preprint)
            node.reload()
            preprint.reload()
{'fraction_non_alphanumeric': 0.11210042329586922, 'fraction_numerical': 0.0005838563713326521, 'mean_word_length': 2.7178513293543136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5106523', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1820, 'n_words': 421}