<commit_before># -*- coding: utf-8 -*-
import sys
import logging
from website.app import setup_django, init_app
from scripts import utils as script_utils
from django.db import transaction

setup_django()
from django.db.models import Q
from osf.models import AbstractNode, Guid, Comment
from addons.wiki.models import NodeWikiPage, WikiPage, WikiVersion
from django.contrib.contenttypes.models import ContentType

logger = logging.getLogger(__name__)

# Replace with output of determine_comments_viewed_timestamps_that_should_change
expected_values = []

def determine_comments_that_should_be_repointed():
    """
    RUN PRE-MIGRATION
    """
    comments_count_root_target = 0
    comments_count_target = 0
    for node in AbstractNode.objects.exclude(wiki_pages_versions={}):
        for key, version_list in node.wiki_pages_versions.iteritems():
            for version in version_list:
                nwp_guid = Guid.load(version)
                comments_count_root_target += Comment.objects.filter(Q(root_target=nwp_guid)).count()
                comments_count_target += Comment.objects.filter(Q(target=nwp_guid)).count()
    print "Comments with root targets as NWP that should be repointed: {}".format(comments_count_root_target)
    print "Comments with NWP targets that should be repointed: {}".format(comments_count_target)
    return comments_count_root_target, comments_count_target

def determine_comments_viewed_timestamps_that_should_change():
    """
    RUN AFTER CREATING WIKI PAGES AND WIKI VERSIONS BUT BEFORE MODIFYING comments_viewed_timestamps, if using
    Save results under expected_values.
    """
    pending_changes = []
    for node in AbstractNode.objects.exclude(wiki_pages_versions={}):
        for key, version_list in node.wiki_pages_versions.iteritems():
            nwp_guid  = version_list[-1]
            nwp_page_name = NodeWikiPage.objects.get(former_guid=nwp_guid).page_name
            wp_guid = node.wikis.get(page_name=nwp_page_name)._id
            for user in node.contributors.all():
                if nwp_guid in user.comments_viewed_timestamp:
                    pending_changes.append({"user_id": user.id, "nwp_guid": str(nwp_guid), "wp_guid": str(wp_guid), "timestamp": str(user.comments_viewed_timestamp.get(nwp_guid)), "cvt_length": len(user.comments_viewed_timestamp.keys())})
    print pending_changes
    return pending_changes

def count_node_wiki_pages():
    """
    RUN POST-MIGRATION.
    Assert that counts for WikiPages, WikiVersions, WikiPage guids are correct.
    Also check that NodeWikiPage guids with an _id of None is correct (should have been repointed to WikiPages)
    """
    nodes_with_wikis = AbstractNode.objects.exclude(wiki_pages_versions={}).exclude(type='osf.collection').exclude(type='osf.quickfilesnode')
    wp_content_type_id = ContentType.objects.get_for_model(WikiPage).id
    wiki_page_count = 0
    wiki_version_counts = 0
    for node in nodes_with_wikis:
        for wiki_key, version_list in node.wiki_pages_versions.iteritems():
            wiki_page_count += 1
            wiki_version_counts += len(version_list)
    assert wiki_page_count == WikiPage.objects.count()
    assert wiki_version_counts == WikiVersion.objects.count()
    assert wiki_page_count + wiki_version_counts == Guid.objects.filter(content_type_id=wp_content_type_id).count()
    assert wiki_version_counts == NodeWikiPage.objects.filter(guids___id=None).count()

def verify_comments_viewed_timestamp():
    """
    RUN POST MIGRATION.
    determine_comments_viewed_timestamps_that_should_change should have been run previously.
    """
    for user_dict in expected_values:
        u = OSFUser.objects.get(id=user_dict['user_id'])
        assert user_dict['nwp_guid'] not in u.comments_viewed_timestamp.keys()
        assert user_dict['wp_guid'] in u.comments_viewed_timestamp.keys()
        assert user_dict['timestamp'] == str(u.comments_viewed_timestamp[user_dict['wp_guid']])
        assert user_dict['cvt_length'] == len(u.comments_viewed_timestamp.keys())

def verify_comments_were_repointed_to_wps():
    """
    RUN POST-MIGRATION
    """
    wp_content_type_id = ContentType.objects.get_for_model(WikiPage).id
    print "Comments with WikiPage root_targets: {}".format(Comment.objects.filter(root_target__content_type_id=wp_content_type_id).count())
    print "Comments with WikiPage targets: {}".format(Comment.objects.filter(target__content_type_id=wp_content_type_id).count())

def main(dry=True):
    init_app(routes=False)
    count_node_wiki_pages()

if __name__ == '__main__':
    dry = '--dry' in sys.argv
    if not dry:
        script_utils.add_file_logger(logger, __file__)
    # Finally run the migration
    main(dry=dry)<commit_msg>Fixes and improvements to verification script<commit_after># -*- coding: utf-8 -*-
import sys
import logging
from website.app import setup_django
from scripts import utils as script_utils

setup_django()
from django.db import connection
from osf.models import AbstractNode, Guid, Comment
from addons.wiki.models import NodeWikiPage, WikiPage, WikiVersion
from django.contrib.contenttypes.models import ContentType

logger = logging.getLogger(__name__)

def determine_comments_that_should_be_repointed():
    """
    RUN PRE-MIGRATION
    """
    logger.debug('Counting Comment records that will be updated...')
    with connection.cursor() as cursor:
        cursor.execute('''
        -- Flatten all nodes' wiki_pages_versions
        CREATE OR REPLACE TEMPORARY VIEW wiki_page_version_guids AS (
            SELECT
                trim(nwp_guid::text, '"') as _id
            FROM osf_abstractnode as oan,
                jsonb_each(oan.wiki_pages_versions) as wiki_pages_versions,
                jsonb_array_elements(wiki_pages_versions->wiki_pages_versions.key) as nwp_guid
        );

        ''')

        cursor.execute('''
        -- Count comments whose root_target points to a GUID in wiki_pages_versions
        SELECT
        COUNT(*)
        FROM osf_comment
        INNER JOIN osf_guid ON (osf_comment.root_target_id = osf_guid.id)
        WHERE osf_guid._id IN (SELECT _id from wiki_page_version_guids);
        ''')
        comments_root_target_count = cursor.fetchone()[0]
        logger.info("Comments with root targets as NWP that should be repointed: {}".format(comments_root_target_count))

        cursor.execute('''
        -- Count comments whose root_target points to a GUID in wiki_pages_versions
        SELECT
        COUNT(*)
        FROM osf_comment
        INNER JOIN osf_guid ON (osf_comment.target_id = osf_guid.id)
        WHERE osf_guid._id IN (SELECT _id from wiki_page_version_guids);
        ''')
        comments_target_count = cursor.fetchone()[0]
        logger.info("Comments with NWP targets that should be repointed: {}".format(comments_target_count))

    return comments_root_target_count, comments_target_count


def count_node_wiki_pages():
    """
    RUN POST-MIGRATION.
    Assert that counts for WikiPages, WikiVersions, WikiPage guids are correct.
    Also check that NodeWikiPage guids with an _id of None is correct (should have been repointed to WikiPages)
    """
    logger.debug('Counting WikiPages, WikiVersions, and WikiPage GUIDS...')
    wiki_pages_versions = AbstractNode.objects.exclude(wiki_pages_versions={}).values_list('wiki_pages_versions', flat=True)
    wp_content_type_id = ContentType.objects.get_for_model(WikiPage).id
    wiki_key_count = 0
    expected_wiki_ids = []
    for wpv in wiki_pages_versions:
        for wiki_key, version_list in wpv.iteritems():
            # expected_wiki_ids.append(wiki_key)
            # Skip bad data: wiki pages that have no versions
            if version_list:
                expected_wiki_ids.extend(version_list)
                wiki_key_count += 1
    wiki_values_count = len(expected_wiki_ids)
    # There will be dupes for nodes that don't have the wiki addon (55 on production).
    # When nodes without the wiki addon are registered, their wiki_pages_versions
    # field gets copied as is.
    wiki_values_count_without_dupes = len(set(expected_wiki_ids))
    wiki_page_count = WikiPage.objects.count()
    assert wiki_key_count == wiki_page_count, (
        'WikiPage count mismatch: # keys in Node.wiki_pages_versions: {}, WikiPage.objects.count(): {}, Difference: {}'.format(
            wiki_key_count,
            wiki_page_count,
            wiki_key_count - wiki_page_count)
    )
    wiki_version_count = WikiVersion.objects.count()
    assert wiki_values_count == wiki_version_count, (
        'WikiVersion count mismatch: # items in Node.wiki_pages_versions.values(): {}, WikiVersion.objects.count(): {}, Difference: {}'.format(
            wiki_values_count,
            wiki_version_count,
            wiki_values_count - wiki_version_count)
    )
    guid_count = Guid.objects.filter(content_type_id=wp_content_type_id).count()
    expected_guid_count = wiki_key_count + wiki_values_count_without_dupes
    assert expected_guid_count == guid_count, (
        'Guid count mismatch: {}, {}, Difference: {}'.format(
            expected_guid_count, guid_count, expected_guid_count - guid_count
        )
    )
    # NodeWikiPages whose Guid was repointed to a WikiPage
    node_wiki_page_count = NodeWikiPage.objects.filter(guids___id=None).count()
    assert wiki_values_count_without_dupes == node_wiki_page_count, (
        'NodeWikiPage count mismatch: # items in Node.wiki_pages_versions: {}, NodeWikiPage.objects.count(): {}, Difference: {}'.format(
            wiki_values_count_without_dupes,
            node_wiki_page_count,
            wiki_values_count_without_dupes - node_wiki_page_count
        )
    )
    logger.info('SUCCESS! WikiPage, WikiVersion, WikiPage GUI and NodeWikiPage counts matched!')

def verify_comments_were_repointed_to_wps():
    """
    RUN POST-MIGRATION
    """
    logger.debug('Counting comments...')
    wp_content_type_id = ContentType.objects.get_for_model(WikiPage).id
    logger.info("Comments with WikiPage root_targets: {}".format(Comment.objects.filter(root_target__content_type_id=wp_content_type_id).count()))
    logger.info("Comments with WikiPage targets: {}".format(Comment.objects.filter(target__content_type_id=wp_content_type_id).count()))
    logger.info('Check the above values against the pre-migration values.')

def main(dry=True):
    pre_migration = '--pre' in sys.argv
    if pre_migration:
        determine_comments_that_should_be_repointed()
    else:
        count_node_wiki_pages()
        verify_comments_were_repointed_to_wps()

if __name__ == '__main__':
    dry = '--dry' in sys.argv
    if not dry:
        script_utils.add_file_logger(logger, __file__)
    # Finally run the migration
    main(dry=dry)
{'fraction_non_alphanumeric': 0.0889282056057361, 'fraction_numerical': 0.001303659558618121, 'mean_word_length': 3.7417218543046356, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3136168', 'n_tokens_mistral': 3382, 'n_tokens_neox': 3216, 'n_words': 713}