<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os

import luigi
from luigi import LocalTarget

from tasks.config import db
from tasks.config import output

from rnacentral.db import cursor
from rnacentral.xml import exporter


class SearchChunkTask(luigi.Task):  # pylint: disable=R0904
    """
    This is a task that will create an xml export for the given range of UPI's.
    """

    min = luigi.IntParameter()
    max = luigi.IntParameter()

    def output(self):
        config = output()
        filepattern = 'xml_export_{min}_{max}.xml'.format(
            min=self.min,
            max=self.max,
        )
        filename = os.path.join(config.search_files, filepattern)
        return LocalTarget(filename)

    def run(self):
        with self.output().open(format='gzip') as raw:
            with cursor(db()) as cur:
                results = exporter.range(cur, self.min, self.max)
                exporter.write(raw, results)<commit_msg>Remove attempt to gzip<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os

import luigi
from luigi import LocalTarget

from tasks.config import db
from tasks.config import output

from rnacentral.db import cursor
from rnacentral.xml import exporter


class SearchChunkTask(luigi.Task):  # pylint: disable=R0904
    """
    This is a task that will create an xml export for the given range of UPI's.
    """

    min = luigi.IntParameter()
    max = luigi.IntParameter()

    def output(self):
        config = output()
        filepattern = 'xml_export_{min}_{max}.xml'.format(
            min=self.min,
            max=self.max,
        )
        filename = os.path.join(config.search_files, filepattern)
        return LocalTarget(filename)

    def run(self):
        with self.output().open('w') as raw:
            with cursor(db()) as cur:
                results = exporter.range(cur, self.min, self.max)
                exporter.write(raw, results)
{'fraction_non_alphanumeric': 0.08725267596496919, 'fraction_numerical': 0.011028219266947779, 'mean_word_length': 3.1340482573726542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3727023', 'n_tokens_mistral': 884, 'n_tokens_neox': 832, 'n_words': 328}