<commit_before>"""
Import OLX data into LORE.
"""

from __future__ import unicode_literals

from shutil import rmtree
import logging
from tempfile import mkdtemp
from os.path import join, exists
from os import remove, listdir

from xbundle import XBundle, DESCRIPTOR_TAGS
from lxml import etree
from archive import extract, ArchiveException

from learningresources.api import create_course, create_resource

log = logging.getLogger(__name__)


def import_course_from_file(filename, repo_id, user_id):
    """
    Import OLX from .zip or tar.gz. Imports from a file
    and deletes the file.

    Args:
        filename (unicode): Path to archive file (zip or .tar.gz)
        repo_id (int): Primary key of repository course belongs to
        user_id (int): Primary key of user importing the course

    Raises:
        ValueError: Unable to extract or read archive contents.

    Returns: None
    """
    tempdir = mkdtemp()
    try:
        extract(path=filename, to_path=tempdir, method="safe")
    except ArchiveException as ex:
        log.debug("failed to extract: %s", ex)
        remove(filename)
        raise ValueError("Invalid OLX archive, unable to extract.")
    course_imported = False
    if "course.xml" in listdir(tempdir):
        import_course_from_path(tempdir, repo_id, user_id)
        course_imported = True
    else:
        for path in listdir(tempdir):
            if exists(join(tempdir, path, 'course.xml')):
                import_course_from_path(join(tempdir, path), repo_id, user_id)
                course_imported = True
    rmtree(tempdir)
    remove(filename)
    if course_imported is False:
        raise ValueError("Invalid OLX archive, no courses found.")


def import_course_from_path(path, repo_id, user_id):
    """
    Import course from an OLX directory.

    Args:
        path (unicode): path to extracted OLX tree
        user_id (int): pk of Django user doing the import
    """
    bundle = XBundle()
    bundle.import_from_directory(path)
    return import_course(bundle, repo_id, user_id)


def import_course(bundle, repo_id, user_id):
    """
    Import a course from an XBundle object.

    Args:
        bundle (xbundle.XBundle): Course as xbundle XML
        user_id (int): pk of Django user doing the import
    """
    src = bundle.course
    course = create_course(
        org=src.attrib["org"],
        repo_id=repo_id,
        course_number=src.attrib["course"],
        run=src.attrib["semester"],
        user_id=user_id,
    )
    import_children(course, src, None)


def import_children(course, element, parent):
    """
    Create LearningResource instances for each element
    of an XML tree.

    Args:
        course (learningresources.Course): Course
        element (lxml.etree): XML element within xbundle
        parent (learningresources.LearningResource): parent LearningResource
    """
    mpath = etree.ElementTree(element).getpath(element)
    resource = create_resource(
        course=course, parent=parent, resource_type=element.tag,
        title=element.attrib.get("display_name", "MISSING"),
        content_xml=etree.tostring(element),
        mpath=mpath,
    )
    for child in element.getchildren():
        if child.tag in DESCRIPTOR_TAGS:
            import_children(course, child, resource)
<commit_msg>updates comment<commit_after>"""
Import OLX data into LORE.
"""

from __future__ import unicode_literals

from shutil import rmtree
import logging
from tempfile import mkdtemp
from os.path import join, exists
from os import remove, listdir

from xbundle import XBundle, DESCRIPTOR_TAGS
from lxml import etree
from archive import extract, ArchiveException

from learningresources.api import create_course, create_resource

log = logging.getLogger(__name__)


def import_course_from_file(filename, repo_id, user_id):
    """
    Import OLX from .zip or tar.gz. Imports from a file
    and deletes the file.

    Args:
        filename (unicode): Path to archive file (zip or .tar.gz)
        repo_id (int): Primary key of repository course belongs to
        user_id (int): Primary key of user importing the course

    Raises:
        ValueError: Unable to extract or read archive contents.

    Returns: None

    A valid OLX archive has a single occurrence of the file course.xml in its
    root directory, or no course.xml in its root and a single occurrence of
    course.xml in one or more of the root directory's children.
    """
    tempdir = mkdtemp()
    try:
        extract(path=filename, to_path=tempdir, method="safe")
    except ArchiveException as ex:
        log.debug("failed to extract: %s", ex)
        remove(filename)
        raise ValueError("Invalid OLX archive, unable to extract.")
    course_imported = False
    if "course.xml" in listdir(tempdir):
        import_course_from_path(tempdir, repo_id, user_id)
        course_imported = True
    else:
        for path in listdir(tempdir):
            if exists(join(tempdir, path, 'course.xml')):
                import_course_from_path(join(tempdir, path), repo_id, user_id)
                course_imported = True
    rmtree(tempdir)
    remove(filename)
    if course_imported is False:
        raise ValueError("Invalid OLX archive, no courses found.")


def import_course_from_path(path, repo_id, user_id):
    """
    Import course from an OLX directory.

    Args:
        path (unicode): path to extracted OLX tree
        user_id (int): pk of Django user doing the import
    """
    bundle = XBundle()
    bundle.import_from_directory(path)
    return import_course(bundle, repo_id, user_id)


def import_course(bundle, repo_id, user_id):
    """
    Import a course from an XBundle object.

    Args:
        bundle (xbundle.XBundle): Course as xbundle XML
        user_id (int): pk of Django user doing the import
    """
    src = bundle.course
    course = create_course(
        org=src.attrib["org"],
        repo_id=repo_id,
        course_number=src.attrib["course"],
        run=src.attrib["semester"],
        user_id=user_id,
    )
    import_children(course, src, None)


def import_children(course, element, parent):
    """
    Create LearningResource instances for each element
    of an XML tree.

    Args:
        course (learningresources.Course): Course
        element (lxml.etree): XML element within xbundle
        parent (learningresources.LearningResource): parent LearningResource
    """
    mpath = etree.ElementTree(element).getpath(element)
    resource = create_resource(
        course=course, parent=parent, resource_type=element.tag,
        title=element.attrib.get("display_name", "MISSING"),
        content_xml=etree.tostring(element),
        mpath=mpath,
    )
    for child in element.getchildren():
        if child.tag in DESCRIPTOR_TAGS:
            import_children(course, child, resource)

{'fraction_non_alphanumeric': 0.08358996920369556, 'fraction_numerical': 0.0, 'mean_word_length': 3.035502958579882, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357258', 'n_tokens_mistral': 2025, 'n_tokens_neox': 1951, 'n_words': 624}