<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import json

import luigi
from luigi import LocalTarget
from luigi.local_target import atomic_file

import attr

from utils import grouper

from tasks.config import mgi
from tasks.mgi.download import MgiDownload
from databases.mgi.parser import rna_entries


class MgiToJson(luigi.Task):  # pylint: disable=R0904
    """
    This will convert the downloaded MGI data to a JSON file containing more
    structured data for the RNA entries.
    """

    def requires(self):
        return MgiDownload()

    def chunk_filename(self, chunk):
        """
        This computes the name of the file for the given chunk to write to.
        """

        filename = self.requires().output().fn
        base = os.path.dirname(filename)
        return os.path.join(base, mgi().json_filename + "_%03i.json" % chunk)

    def output(self):
        first_chunk = self.chunk_filename(0)
        return LocalTarget(first_chunk)

    def run(self):
        input_file = self.requires().output().fn
        entries = list(rna_entries(input_file))
        grouped = grouper(entries, mgi().max_entry_count)
        for index, group in enumerate(grouped):
            data = []
            filename = self.chunk_filename(index)
            for entry in group:
                if not entry:
                    continue
                result = attr.asdict(entry)
                result['feature_type'] = entry.feature_type
                result['ncrna_class'] = entry.ncrna_class
                result['feature_location_start'] = entry.feature_location_start
                result['feature_location_end'] = entry.feature_location_end
                result['external_id'] = entry.accession
                for index, reference in enumerate(entry.references):
                    result['references'][index]['md5'] = reference.md5()

                data.append(result)

            with atomic_file(filename) as out:
                json.dump(data, out)<commit_msg>Mgi should not batch JSON files<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import json

import luigi
from luigi import LocalTarget
from luigi.local_target import atomic_file

import attr

from tasks.config import mgi
from tasks.mgi.download import MgiDownload
from databases.mgi.parser import rna_entries


class MgiToJson(luigi.Task):  # pylint: disable=R0904
    """
    This will convert the downloaded MGI data to a JSON file containing more
    structured data for the RNA entries.
    """

    def requires(self):
        return MgiDownload()

    def output(self):
        downloaded = self.requires().output().fn
        base = os.path.dirname(downloaded)
        filename = os.path.join(base, mgi().json_filename)
        return LocalTarget(filename)

    def run(self):
        input_file = self.requires().output().fn
        data = []
        for entry in rna_entries(input_file):
            if not entry:
                continue
            result = attr.asdict(entry)
            result['feature_type'] = entry.feature_type
            result['ncrna_class'] = entry.ncrna_class
            result['feature_location_start'] = entry.feature_location_start
            result['feature_location_end'] = entry.feature_location_end
            result['external_id'] = entry.accession
            for index, reference in enumerate(entry.references):
                result['references'][index]['md5'] = reference.md5()

            data.append(result)

        filename = self.output().fn
        with atomic_file(filename) as out:
            json.dump(data, out)
{'fraction_non_alphanumeric': 0.08436776772564696, 'fraction_numerical': 0.008626130864717021, 'mean_word_length': 2.8062449959967974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1044999', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1236, 'n_words': 425}