<commit_before>import csv
import io
import logging
import ntpath
import os
import sys
from collections import OrderedDict
from functools import partial
from tempfile import mkstemp

from django.core.management.base import CommandError
from django.db.models import OuterRef
from django.db.models import Q
from django.db.models import Subquery

from kolibri.core.auth.constants import role_kinds
from kolibri.core.auth.constants.collection_kinds import CLASSROOM
from kolibri.core.auth.models import Facility
from kolibri.core.auth.models import FacilityUser
from kolibri.core.auth.models import Role
from kolibri.core.query import GroupConcat
from kolibri.core.tasks.management.commands.base import AsyncCommand
from kolibri.core.tasks.utils import get_current_job
from kolibri.utils import conf

try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError

logger = logging.getLogger(__name__)

# TODO: decide whether these should be internationalized
labels = OrderedDict(
    (
        ("username", "Username"),
        ("password", "Password"),
        ("full_name", "Full name"),
        ("kind", "User type"),
        ("id_number", "ID number"),
        ("birth_year", "Birth year"),
        ("gender", "Gender"),
        ("assigned", "Enrolled in"),
        ("enrolled", "Assigned to"),
    )
)

db_columns = (
    "username",
    "id",
    # "password",
    "full_name",
    "birth_year",
    "gender",
    "id_number",
    "kind",
    "assigned",
    "enrolled",
)
# These constants must be entered vertbatim in the CSV
roles_map = {
    role_kinds.ADMIN: "ADMIN",
    role_kinds.COACH: "COACH",
    role_kinds.ASSIGNABLE_COACH: "ASSIGNABLE_COACH",<commit_msg>Fix header name<commit_after>import csv
import io
import logging
import ntpath
import os
import sys
from collections import OrderedDict
from functools import partial
from tempfile import mkstemp

from django.core.management.base import CommandError
from django.db.models import OuterRef
from django.db.models import Q
from django.db.models import Subquery

from kolibri.core.auth.constants import role_kinds
from kolibri.core.auth.constants.collection_kinds import CLASSROOM
from kolibri.core.auth.models import Facility
from kolibri.core.auth.models import FacilityUser
from kolibri.core.auth.models import Role
from kolibri.core.query import GroupConcat
from kolibri.core.tasks.management.commands.base import AsyncCommand
from kolibri.core.tasks.utils import get_current_job
from kolibri.utils import conf

try:
    FileNotFoundError
except NameError:
    FileNotFoundError = IOError

logger = logging.getLogger(__name__)

# TODO: decide whether these should be internationalized
labels = OrderedDict(
    (
        ("username", "Username"),
        ("password", "Password"),
        ("full_name", "Full name"),
        ("kind", "User type"),
        ("id_number", "Identifier"),
        ("birth_year", "Birth year"),
        ("gender", "Gender"),
        ("assigned", "Enrolled in"),
        ("enrolled", "Assigned to"),
    )
)

db_columns = (
    "username",
    "id",
    # "password",
    "full_name",
    "birth_year",
    "gender",
    "id_number",
    "kind",
    "assigned",
    "enrolled",
)
# These constants must be entered vertbatim in the CSV
roles_map = {
    role_kinds.ADMIN: "ADMIN",
    role_kinds.COACH: "COACH",
    role_kinds.ASSIGNABLE_COACH: "ASSIGNABLE_COACH",
{'fraction_non_alphanumeric': 0.1049510821227394, 'fraction_numerical': 0.0, 'mean_word_length': 3.9040697674418605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1886143', 'n_tokens_mistral': 1044, 'n_tokens_neox': 1035, 'n_words': 288}