<commit_before>import os
from django.conf import settings
from tosec import models
from tosec.parser import TosecParser
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    args = '<tosec_catalog tosec_catalog ...>'
    help = "Import Tosec catalog files in the database"

    def handle(self, *args, **kwargs):
        if args:
            filenames = args
        else:
            basepath = os.path.join(settings.TOSEC_DAT_PACK, 'TOSEC')
            if not os.path.exists(basepath):
                self.stderr.write("No TOSEC database found in %s" % basepath)
                return
            filenames = [os.path.join(basepath, f)
                         for f in os.listdir(basepath)]
        for filename in filenames:
            self.stdout.write("Importing %s" % filename)
            with open(filename, 'r') as tosec_dat:
                dat_contents = tosec_dat.readlines()

            tosec_parser = TosecParser(dat_contents)
            tosec_parser.parse()

            category = models.Category(
                name=tosec_parser.headers['name'],
                description=tosec_parser.headers['description'],
                category=tosec_parser.headers['category'],
                version=tosec_parser.headers['version'],
                author=tosec_parser.headers['author'],
            )
            category.save()

            for game in tosec_parser.games:
                game_row = models.Game(
                    category=category,
                    name=game['name'],
                    description=game['description'],
                )
                game_row.save()<commit_msg>Track progress of number of TOSEC dats imported<commit_after>import os
from django.conf import settings
from tosec import models
from tosec.parser import TosecParser
from django.core.management.base import BaseCommand


class Command(BaseCommand):
    args = '<tosec_catalog tosec_catalog ...>'
    help = "Import Tosec catalog files in the database"

    def handle(self, *args, **kwargs):
        if args:
            filenames = args
        else:
            basepath = os.path.join(settings.TOSEC_DAT_PACK, 'TOSEC')
            if not os.path.exists(basepath):
                self.stderr.write("No TOSEC database found in %s" % basepath)
                return
            filenames = [os.path.join(basepath, f)
                         for f in os.listdir(basepath)]
        total_files = len(filenames)
        for index, filename in enumerate(filenames, start=1):
            self.stdout.write("Importing {} [{} of {}]".format(filename,
                                                               index,
                                                               total_files))
            with open(filename, 'r') as tosec_dat:
                dat_contents = tosec_dat.readlines()

            tosec_parser = TosecParser(dat_contents)
            tosec_parser.parse()

            category = models.Category(
                name=tosec_parser.headers['name'],
                description=tosec_parser.headers['description'],
                category=tosec_parser.headers['category'],
                version=tosec_parser.headers['version'],
                author=tosec_parser.headers['author'],
            )
            category.save()

            for game in tosec_parser.games:
                game_row = models.Game(
                    category=category,
                    name=game['name'],
                    description=game['description'],
                )
                game_row.save()
{'fraction_non_alphanumeric': 0.0925356443947442, 'fraction_numerical': 0.0002795638803466592, 'mean_word_length': 1.696307460437076, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5702087', 'n_tokens_mistral': 930, 'n_tokens_neox': 902, 'n_words': 204}