# Path: languk/corpus/ud_converter.py
# COMPRESS_UPOS_MAPPING = dict(_UPOS_MAPPING)
# 
# def compress_features(feats):
#     res = ""
#     for pair in feats.split("|"):
#         if not pair:
#             continue
#         cat, val = pair.split("=")
# 
#         try:
#             c_cat = COMPRESS_FEATURES_MAPPING[cat]
#         except KeyError:
#             logger.warning(f"Cannot find the feature '{cat}' in the mapping, skipping it for now")
#             continue
# 
#         try:
#             c_val = COMPRESS_FEATURE_VALUES_MAPPING[cat][val]
#         except KeyError:
#             logger.warning(f"Cannot find the value '{val}' for the feature '{cat}' in the mapping, skipping it for now")
#             continue
# 
#         res += c_cat + c_val
# 
#     return res
# 
# def decompress(tokens: str=None, ud_lemmas: str=None, ud_features: str=None, ud_postags: str=None) -> List[OrderedDict]:
#     params = locals()
# 
#     assert any(
#         map(lambda x: x is not None, params.values())
#     ), "at least one param should be not None"
# 
#     zipped: dict = {}
# 
#     for param_name, param_value in params.items():
#         if param_value is not None:
# #             if param_name == "tokens":
# #                 # TODO: validate if this workaround can be properly fixed
# #                 param_value = param_value.strip()
#             zipped[param_name] = unpack_values(param_name, param_value)
#     
# 
#     sentences_length: set = set(map(len, zipped.values()))
#     assert len(sentences_length) == 1, f"Text contains different number of sentences: {sentences_length}"
# 
#     res: list = []
#     param_names: list[str] = list(zipped.keys())
#     param_values: list[str] = list(zipped.values())
# 
#     for sent in zip(*param_values): 
#         word_length:set = set(map(len, sent))
# 
#         assert len(sentences_length) == 1, f"Text contains different number of words in sentence: {sent}"
# 
#         res.append(
#             [OrderedDict(zip(param_names, word_info)) for word_info in zip(*sent)]
#         )
# 
# 
#     return res
# 
# Path: languk/corpus/models.py
# class ExportCorpusTask(TaskRQ):
#     file_format = models.CharField(
#         max_length=5, null=False, blank=False, default="txt", choices=(("txt", "Text File"),)
#     )
#     file_compression = models.CharField(
#         max_length=5,
#         null=False,
#         blank=False,
#         default="none",
#         choices=(
#             ("none", "No compression"),
#             ("bz2", "Bzip2"),
#             ("lzma", "LZMA"),
#         ),
#     )
# 
#     corpora = ChoiceArrayField(
#         models.CharField(
#             max_length=10,
#             null=False,
#             blank=False,
#             choices=_CORPORA_CHOICES,
#         ),
#         blank=False,
#     )
# 
#     filtering = ChoiceArrayField(
#         models.CharField(
#             max_length=10,
#             null=False,
#             blank=False,
#             choices=_FILTERING_CHOICES,
#         ),
#         blank=True,
#     )
# 
#     processing = models.CharField(
#         max_length=10,
#         null=False,
#         blank=False,
#         choices=(
#             ("orig", "Original texts in markdown format"),
#             ("tokens", "Tokenized by NLP-UK lib"),
#             ("lemmas", "Lemmatized by NLP-UK lib"),
#         ),
#     )
# 
#     TASK_QUEUE = settings.QUEUE_DEFAULT
# 
#     DEFAULT_VERBOSITY = 2
#     TASK_TIMEOUT = 0
#     LOG_TO_FIELD = True
#     LOG_TO_FILE = False
# 
#     @staticmethod
#     def get_jobclass():
#         from .jobs import ExportCorpusJob
# 
#         return ExportCorpusJob
# 
# _CORPORA_CHOICES: Tuple[Tuple[str, str]] = (
#     ("news", "News and magazines"),
#     ("wikipedia", "Ukrainian Wikipedia"),
#     ("fiction", "Fiction"),
#     ("court", "Sampled court decisions"),
#     ("laws", "Laws and bylaws"),
# )


import logging
import bz2
import lzma
import os.path
import pathlib
import csv
import pymongo
from collections import defaultdict, Counter
from typing import TextIO
from django.conf import settings
from django_task.job import Job
from .ud_converter import COMPRESS_UPOS_MAPPING, compress_features, decompress
from .models import ExportCorpusTask, _CORPORA_CHOICES
        from .mongodb import get_db
        from .mongodb import get_db
        from .udpipe_model import Model as UDPipeModel
        from ufal.udpipe import Sentence  # type: ignore
        from .mongodb import get_db
                    if "tokens" not in article["nlp"][f]:
                        task.log(
                            logging.WARNING,
                            f"Cannot find tokenized version of field {f} in the document {article['_id']}",
                        )
                        continue

                    for s in article["nlp"][f]["tokens"].split("\n"):
                        # ignoring default model tokenizer in order to use whitespace tokenizer

                        tok_sent = Sentence()
                        for w in s.split(" "):
                            tok_sent.addWord(w)

                        sent_lemmas = []
                        sent_postags = []
                        sent_features = []

                        model.tag(tok_sent)

                        for w in tok_sent.words[1:]:
                            poses.update([w.upostag])
                            sent_lemmas.append(w.lemma)
                            # Again, not moving that to a separate function to
                            # reduce number of unnecessary calls
                            try:
                                sent_postags.append(COMPRESS_UPOS_MAPPING[w.upostag])
                            except KeyError:
                                task.log(
                                    logging.WARNING,

# Path: languk/corpus/ud_converter.py
# COMPRESS_UPOS_MAPPING = dict(_UPOS_MAPPING)
# 
# def compress_features(feats):
#     res = ""
#     for pair in feats.split("|"):
#         if not pair:
#             continue
#         cat, val = pair.split("=")
# 
#         try:
#             c_cat = COMPRESS_FEATURES_MAPPING[cat]
#         except KeyError:
#             logger.warning(f"Cannot find the feature '{cat}' in the mapping, skipping it for now")
#             continue
# 
#         try:
#             c_val = COMPRESS_FEATURE_VALUES_MAPPING[cat][val]
#         except KeyError:
#             logger.warning(f"Cannot find the value '{val}' for the feature '{cat}' in the mapping, skipping it for now")
#             continue
# 
#         res += c_cat + c_val
# 
#     return res
# 
# def decompress(tokens: str=None, ud_lemmas: str=None, ud_features: str=None, ud_postags: str=None) -> List[OrderedDict]:
#     params = locals()
# 
#     assert any(
#         map(lambda x: x is not None, params.values())
#     ), "at least one param should be not None"
# 
#     zipped: dict = {}
# 
#     for param_name, param_value in params.items():
#         if param_value is not None:
# #             if param_name == "tokens":
# #                 # TODO: validate if this workaround can be properly fixed
# #                 param_value = param_value.strip()
#             zipped[param_name] = unpack_values(param_name, param_value)
#     
# 
#     sentences_length: set = set(map(len, zipped.values()))
#     assert len(sentences_length) == 1, f"Text contains different number of sentences: {sentences_length}"
# 
#     res: list = []
#     param_names: list[str] = list(zipped.keys())
#     param_values: list[str] = list(zipped.values())
# 
#     for sent in zip(*param_values): 
#         word_length:set = set(map(len, sent))
# 
#         assert len(sentences_length) == 1, f"Text contains different number of words in sentence: {sent}"
# 
#         res.append(
#             [OrderedDict(zip(param_names, word_info)) for word_info in zip(*sent)]
#         )
# 
# 
#     return res
# 
# Path: languk/corpus/models.py
# class ExportCorpusTask(TaskRQ):
#     file_format = models.CharField(
#         max_length=5, null=False, blank=False, default="txt", choices=(("txt", "Text File"),)
#     )
#     file_compression = models.CharField(
#         max_length=5,
#         null=False,
#         blank=False,
#         default="none",
#         choices=(
#             ("none", "No compression"),
#             ("bz2", "Bzip2"),
#             ("lzma", "LZMA"),
#         ),
#     )
# 
#     corpora = ChoiceArrayField(
#         models.CharField(
#             max_length=10,
#             null=False,
#             blank=False,
#             choices=_CORPORA_CHOICES,
#         ),
#         blank=False,
#     )
# 
#     filtering = ChoiceArrayField(
#         models.CharField(
#             max_length=10,
#             null=False,
#             blank=False,
#             choices=_FILTERING_CHOICES,
#         ),
#         blank=True,
#     )
# 
#     processing = models.CharField(
#         max_length=10,
#         null=False,
#         blank=False,
#         choices=(
#             ("orig", "Original texts in markdown format"),
#             ("tokens", "Tokenized by NLP-UK lib"),
#             ("lemmas", "Lemmatized by NLP-UK lib"),
#         ),
#     )
# 
#     TASK_QUEUE = settings.QUEUE_DEFAULT
# 
#     DEFAULT_VERBOSITY = 2
#     TASK_TIMEOUT = 0
#     LOG_TO_FIELD = True
#     LOG_TO_FILE = False
# 
#     @staticmethod
#     def get_jobclass():
#         from .jobs import ExportCorpusJob
# 
#         return ExportCorpusJob
# 
# _CORPORA_CHOICES: Tuple[Tuple[str, str]] = (
#     ("news", "News and magazines"),
#     ("wikipedia", "Ukrainian Wikipedia"),
#     ("fiction", "Fiction"),
#     ("court", "Sampled court decisions"),
#     ("laws", "Laws and bylaws"),
# )


# Path: languk/corpus/jobs.py
import logging
import bz2
import lzma
import os.path
import pathlib
import csv
import pymongo
from collections import defaultdict, Counter
from typing import TextIO
from django.conf import settings
from django_task.job import Job
from .ud_converter import COMPRESS_UPOS_MAPPING, compress_features, decompress
from .models import ExportCorpusTask, _CORPORA_CHOICES
        from .mongodb import get_db
        from .mongodb import get_db
        from .udpipe_model import Model as UDPipeModel
        from ufal.udpipe import Sentence  # type: ignore
        from .mongodb import get_db

                    if "tokens" not in article["nlp"][f]:
                        task.log(
                            logging.WARNING,
                            f"Cannot find tokenized version of field {f} in the document {article['_id']}",
                        )
                        continue

                    for s in article["nlp"][f]["tokens"].split("\n"):
                        # ignoring default model tokenizer in order to use whitespace tokenizer

                        tok_sent = Sentence()
                        for w in s.split(" "):
                            tok_sent.addWord(w)

                        sent_lemmas = []
                        sent_postags = []
                        sent_features = []

                        model.tag(tok_sent)

                        for w in tok_sent.words[1:]:
                            poses.update([w.upostag])
                            sent_lemmas.append(w.lemma)
                            # Again, not moving that to a separate function to
                            # reduce number of unnecessary calls
                            try:
                                sent_postags.append(COMPRESS_UPOS_MAPPING[w.upostag])
                            except KeyError:
                                task.log(
                                    logging.WARNING,

                                    f"Cannot find {w.upostag} in the COMPRESS_UPOS_MAPPING, skipping for now",