<commit_before>"""\
Run this to upgrade the site.

Examples

To update on the production server:

    %(prog)s production.ini

For the development.ini you must supply the paster app name:

    %(prog)s development.ini --app-name app

"""
from contextlib import contextmanager
import logging

EPILOG = __doc__

logger = logging.getLogger(__name__)


DEFAULT_COLLECTIONS = [
]


def internal_app(configfile, app_name=None, username=None):
    from webtest import TestApp
    from pyramid import paster
    app = paster.get_app(configfile, app_name)
    if not username:
        username = 'IMPORT'
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': username,
    }
    return TestApp(app, environ)


def run(testapp, collections):
    from ..storage import DBSession
    with AlternateScope(DBSession) as scope:
        if not collections:
            collections = DEFAULT_COLLECTIONS
        root = testapp.app.root_factory(testapp.app)
        for collection_name in collections:
            collection = root[collection_name]
            count = 0
            errors = 0
            logger.info('Upgrading %s', collection_name)
            for uuid in collection:
                count += 1
                with scope.change():
                    try:
                        testapp.patch_json('/%s' % uuid, {})
                    except Exception:
                        logger.exception('Upgrade failed for: /%s/%s', collection_name, uuid)
                        errors += 1
                if count % 1000 == 0:
                    logger.info('Upgrading %s: %d', collection_name, count)
            logger.info('Upgraded %s: %d (errors: %d)', collection_name, count, errors)


class AlternateScope(object):
    def __init__(self, DBSession):
        self.scope = None
        self._DBSession = DBSession

    def __enter__(self):
        import transaction
        from zope.sqlalchemy.datamanager import join_transaction
        from sqlalchemy.orm.scoping import ScopedRegistry
        self._original_registry = self._DBSession.registry
        self._DBSession.registry = ScopedRegistry(
            self._DBSession.session_factory, self._get_scope)
        self.scope = self
        txn = transaction.begin()
        session = self._DBSession()
        join_transaction(session)
        transaction.manager.free(txn)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self._DBSession.registry = self._original_registry
        self.scope = None

    def _get_scope(self):
        return self.scope

    @contextmanager
    def change(self, scope=None):
        previous = self.scope
        self.scope = scope
        yield scope
        self.scope = previous


def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update links and keys", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    testapp = internal_app(args.config_uri, args.app_name)

    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    run(testapp, args.item_type)


if __name__ == '__main__':
    main()
<commit_msg>Added in default collections to upgrade.<commit_after>"""\
Run this to upgrade the site.

Examples

To update on the production server:

    %(prog)s production.ini

For the development.ini you must supply the paster app name:

    %(prog)s development.ini --app-name app

"""
from contextlib import contextmanager
import logging

EPILOG = __doc__

logger = logging.getLogger(__name__)


DEFAULT_COLLECTIONS = [
    antibody_characterization,
    biosample,
    biosample_characterization,
    dataset,
    document,
    donor_characterization,
    experiment,
    human_donor,
    mouse_donor,
    publication
]


def internal_app(configfile, app_name=None, username=None):
    from webtest import TestApp
    from pyramid import paster
    app = paster.get_app(configfile, app_name)
    if not username:
        username = 'IMPORT'
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': username,
    }
    return TestApp(app, environ)


def run(testapp, collections):
    from ..storage import DBSession
    with AlternateScope(DBSession) as scope:
        if not collections:
            collections = DEFAULT_COLLECTIONS
        root = testapp.app.root_factory(testapp.app)
        for collection_name in collections:
            collection = root[collection_name]
            count = 0
            errors = 0
            logger.info('Upgrading %s', collection_name)
            for uuid in collection:
                count += 1
                with scope.change():
                    try:
                        testapp.patch_json('/%s' % uuid, {})
                    except Exception:
                        logger.exception('Upgrade failed for: /%s/%s', collection_name, uuid)
                        errors += 1
                if count % 1000 == 0:
                    logger.info('Upgrading %s: %d', collection_name, count)
            logger.info('Upgraded %s: %d (errors: %d)', collection_name, count, errors)


class AlternateScope(object):
    def __init__(self, DBSession):
        self.scope = None
        self._DBSession = DBSession

    def __enter__(self):
        import transaction
        from zope.sqlalchemy.datamanager import join_transaction
        from sqlalchemy.orm.scoping import ScopedRegistry
        self._original_registry = self._DBSession.registry
        self._DBSession.registry = ScopedRegistry(
            self._DBSession.session_factory, self._get_scope)
        self.scope = self
        txn = transaction.begin()
        session = self._DBSession()
        join_transaction(session)
        transaction.manager.free(txn)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self._DBSession.registry = self._original_registry
        self.scope = None

    def _get_scope(self):
        return self.scope

    @contextmanager
    def change(self, scope=None):
        previous = self.scope
        self.scope = scope
        yield scope
        self.scope = previous


def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update links and keys", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--item-type', action='append', help="Item type")
    parser.add_argument('config_uri', help="path to configfile")
    args = parser.parse_args()

    logging.basicConfig()
    testapp = internal_app(args.config_uri, args.app_name)

    # Loading app will have configured from config file. Reconfigure here:
    logging.getLogger('encoded').setLevel(logging.DEBUG)

    run(testapp, args.item_type)


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.09100998890122086, 'fraction_numerical': 0.0024972253052164264, 'mean_word_length': 2.4775687409551375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6489026', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1995, 'n_words': 513}