<commit_before>"""
Test for assets cleanup of courses for Mac OS metadata files (with filename ".DS_Store"
or with filename which starts with "._")
"""
from django.core.management import call_command

from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.contentstore.content import XASSET_LOCATION_TAG
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.base import location_to_query
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_from_xml


class ExportAllCourses(ModuleStoreTestCase):
    """
    Tests assets cleanup for all courses.
    """
    def setUp(self):
        """ Common setup. """
        self.content_store = contentstore()
        self.module_store = modulestore()

    def test_export_all_courses(self):
        """
        This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
        cleaned up on import
        """
        import_from_xml(
            self.module_store,
            '**replace_user**',
            'common/test/data/',
            ['dot-underscore'],
            static_content_store=self.content_store,
            do_import_static=True,
            verbose=True
        )

        course = self.module_store.get_course(SlashSeparatedCourseKey('edX', 'dot-underscore', '2014_Fall'))
        self.assertIsNotNone(course)

        # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
        self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')

        # manually add redundant assets (file ".DS_Store" and filename starts with "._")
        course_filter = course.id.make_asset_key("asset", None)
        query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG)
        query['_id.name'] = all_assets[0]['_id']['name']
        asset_doc = self.content_store.fs_files.find_one(query)
        asset_doc['_id']['name'] = u'._example_test.txt'
        self.content_store.fs_files.insert(asset_doc)
        asset_doc['_id']['name'] = u'.DS_Store'
        self.content_store.fs_files.insert(asset_doc)

        # check that now course has four assets
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 4)
        self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
        self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')
        self.assertEqual(all_assets[2]['_id']['name'], u'._example_test.txt')
        self.assertEqual(all_assets[3]['_id']['name'], u'.DS_Store')

        # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
        call_command('cleanup_assets')
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
        self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')<commit_msg>Make test_export_all_courses do not rely on the order in which files are returned by os.walk().<commit_after>"""
Test for assets cleanup of courses for Mac OS metadata files (with filename ".DS_Store"
or with filename which starts with "._")
"""
from django.core.management import call_command

from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.contentstore.content import XASSET_LOCATION_TAG
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.base import location_to_query
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_from_xml


class ExportAllCourses(ModuleStoreTestCase):
    """
    Tests assets cleanup for all courses.
    """
    def setUp(self):
        """ Common setup. """
        self.content_store = contentstore()
        self.module_store = modulestore()

    def test_export_all_courses(self):
        """
        This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
        cleaned up on import
        """
        import_from_xml(
            self.module_store,
            '**replace_user**',
            'common/test/data/',
            ['dot-underscore'],
            static_content_store=self.content_store,
            do_import_static=True,
            verbose=True
        )

        course = self.module_store.get_course(SlashSeparatedCourseKey('edX', 'dot-underscore', '2014_Fall'))
        self.assertIsNotNone(course)

        # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))

        # manually add redundant assets (file ".DS_Store" and filename starts with "._")
        course_filter = course.id.make_asset_key("asset", None)
        query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG)
        query['_id.name'] = all_assets[0]['_id']['name']
        asset_doc = self.content_store.fs_files.find_one(query)
        asset_doc['_id']['name'] = u'._example_test.txt'
        self.content_store.fs_files.insert(asset_doc)
        asset_doc['_id']['name'] = u'.DS_Store'
        self.content_store.fs_files.insert(asset_doc)

        # check that now course has four assets
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 4)
        self.assertEqual(
            set([asset['_id']['name'] for asset in all_assets]),
            set([u'.example.txt', u'example.txt', u'._example_test.txt', u'.DS_Store'])
        )
        # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
        call_command('cleanup_assets')
        all_assets, count = self.content_store.get_all_content_for_course(course.id)
        self.assertEqual(count, 2)
        self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))
{'fraction_non_alphanumeric': 0.11556029232643118, 'fraction_numerical': 0.0036540803897685747, 'mean_word_length': 3.480900409276944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5398484', 'n_tokens_mistral': 1961, 'n_tokens_neox': 1905, 'n_words': 413}