<commit_before># -*- coding: utf-8 -*-
import xlwt
from optparse import make_option
from collections import OrderedDict

from django.core.management.base import BaseCommand

from data.models import Provider

class Command(BaseCommand):
    help = "Exports list of OCW courses to XLS and TSV"
    args = "--xls <file> --tsv <file>"

    option_list = BaseCommand.option_list + (
        make_option("--xls", action="store", dest="xls_filename", help="XLS filename"),
    )

    def handle(self, *args, **options):
        if options.get("xls_filename"):
            self.export_xls(options.get("xls_filename"))

    def export_xls(self, xls_filename):
        wbk = xlwt.Workbook()
        sheet = wbk.add_sheet('Courses')

        excel_date_fmt = 'D.M.Y'
        style = xlwt.XFStyle()
        style.num_format_str = excel_date_fmt

        first_row = OrderedDict([
            ('OCW Link', 18000),    # 0
            ('URL Hash', 8000),     # 1
            ('Link', 20000),        # 2
            ('Provider', 8000),     # 3
            ('Language', 5000),     # 4
            ('Tags', 8000),         # 5
            ('Author', 8000),       # 6
            ('Title', 10000),       # 7
            ('Description', 10000), # 8
            ('Published', 5000),    # 9
            ('Indexed', 5000),      # 10
            ('Modified', 5000),     # 11
            ('Categories', 10000)   # 12
        ])

        r = c = 0

        for item in first_row:
            sheet.write(r, c, item)
            sheet.col(c).width = first_row[item]
            c += 1
        r += 1

        link = 'http://www.ocwconsortium.org/courses/view/%s/'

        for provider in Provider.objects.all():
            for source in provider.source_set.active():
                for course in source.course_set.all():

                    sheet.write(r, 0, link % course.linkhash)
                    sheet.write(r, 1, course.linkhash)
                    sheet.write(r, 2, course.linkurl)
                    sheet.write(r, 3, provider.name)
                    sheet.write(r, 4, course.language)
                    sheet.write(r, 5, course.tags)
                    sheet.write(r, 6, course.author)
                    sheet.write(r, 7, course.title)
                    sheet.write(r, 8, course.description[:32760])
                    sheet.write(r, 9, course.date_published, style)
                    sheet.write(r, 10, course.date_indexed, style)
                    sheet.write(r, 11, course.date_modified, style)<commit_msg>Updated course export<commit_after># -*- coding: utf-8 -*-
import xlwt
from optparse import make_option
from collections import OrderedDict

from django.core.management.base import BaseCommand

from data.models import Provider

class Command(BaseCommand):
    help = "Exports list of OCW courses to XLS and TSV"
    args = "--xls <file> --tsv <file>"

    option_list = BaseCommand.option_list + (
        make_option("--xls", action="store", dest="xls_filename", help="XLS filename"),
    )

    def handle(self, *args, **options):
        if options.get("xls_filename"):
            self.export_xls(options.get("xls_filename"))

    def export_xls(self, xls_filename):
        wbk = xlwt.Workbook()
        sheet = wbk.add_sheet('Courses')

        excel_date_fmt = 'D.M.Y'
        style = xlwt.XFStyle()
        style.num_format_str = excel_date_fmt

        first_row = OrderedDict([
            ('OCW Link', 18000),    # 0
            ('URL Hash', 8000),     # 1
            ('Link', 20000),        # 2
            ('Provider', 8000),     # 3
            ('Language', 5000),     # 4
            ('Tags', 8000),         # 5
            ('Author', 8000),       # 6
            ('Title', 10000),       # 7
            ('Description', 10000), # 8
            ('Published', 5000),    # 9
            ('Indexed', 5000),      # 10
            ('Modified', 5000),     # 11
            ('Categories', 10000)   # 12
        ])

        r = c = 0

        for item in first_row:
            sheet.write(r, c, item)
            sheet.col(c).width = first_row[item]
            c += 1
        r += 1

        link = 'http://www.ocwconsortium.org/courses/view/%s/'

        for provider in Provider.objects.filter(active=True):
            for source in provider.source_set.filter(disabled=False):
                for course in source.course_set.filter(is_404=False):
                    print(course.id)

                    sheet.write(r, 0, link % course.linkhash)
                    sheet.write(r, 1, course.linkhash)
                    sheet.write(r, 2, course.linkurl)
                    sheet.write(r, 3, provider.name)
                    sheet.write(r, 4, course.language)
                    sheet.write(r, 5, course.tags)
                    sheet.write(r, 6, course.author)
                    sheet.write(r, 7, course.title)
                    sheet.write(r, 8, course.description[:32760])
                    sheet.write(r, 9, course.date_published, style)
                    sheet.write(r, 10, course.date_indexed, style)
                    sheet.write(r, 11, course.date_modified, style)
{'fraction_non_alphanumeric': 0.12832550860719874, 'fraction_numerical': 0.03814553990610329, 'mean_word_length': 1.642377260981912, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6809295', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1469, 'n_words': 358}