<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os

import psycopg2

import luigi
from luigi.local_target import atomic_file

from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord

from tasks.release.utils.db import get_db_connection
from tasks.config import db


class FastaExportBase(luigi.Task):
    table = None
    populate = None
    fetch = None

    def sequences(self, cursor):
        print(self.fetch)
        cursor.execute(self.fetch)
        print('executed')
        for result in cursor:
            print(result)
            yield SeqRecord(
                Seq(result['sequence']),
                id=result['id'],
                description=result['description'],
            )

    def run(self):
        connection = get_db_connection(db(), connect_timeout=20 * 60)
        # populate_table_by_partition(connection, self.table, self.populate)
        # unique = md5(self.table).\
        #     update(self.populate).\
        #     update(self.fetch).\
        #     hexdigest()

        cursor = connection.cursor(
            cursor_factory=psycopg2.extras.DictCursor,
            name=self.__class__.__name__,
        )
        filename = self.output().fn
        try:
            os.makedirs(os.path.basename(filename))
        except:
            pass

        with atomic_file(filename) as out:
            SeqIO.write(self.sequences(cursor), out, "fasta")
        cursor.close()
        connection.close()


def populate_table_by_partition(connection, table, populate):
    """
    This will create and populate some temporary table using the given table
    create statement and populate statement. The population is done by
    partitioned xref table, and the statement will be formatted with dbid prior
    to being run.
    """

    cursor = connection.cursor()
    cursor.execute(table)
    cursor.execute('select id from rnc_database order by id')
    dbids = [r[0] for r in cursor.fetchall()]
    for dbid in dbids:
        cursor.execute(populate.format(dbid=dbid))
<commit_msg>No sever side cursor and build directories<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os

import psycopg2.extras

import luigi
from luigi.local_target import atomic_file

from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord

from tasks.release.utils.db import get_db_connection
from tasks.config import db


class FastaExportBase(luigi.Task):
    table = None
    populate = None
    fetch = None

    def sequences(self, cursor):
        print(self.fetch)
        cursor.execute(self.fetch)
        print('executed')
        for result in cursor:
            print(result)
            yield SeqRecord(
                Seq(result['sequence']),
                id=result['id'],
                description=result['description'],
            )

    def run(self):
        connection = get_db_connection(db(), connect_timeout=20 * 60)
        # populate_table_by_partition(connection, self.table, self.populate)
        # unique = md5(self.table).\
        #     update(self.populate).\
        #     update(self.fetch).\
        #     hexdigest()

        cursor = connection.cursor(
            cursor_factory=psycopg2.extras.DictCursor,
        )
        filename = self.output().fn
        try:
            os.makedirs(os.path.dirname(filename))
        except:
            pass

        with atomic_file(filename) as out:
            SeqIO.write(self.sequences(cursor), out, "fasta")
        cursor.close()
        connection.close()


def populate_table_by_partition(connection, table, populate):
    """
    This will create and populate some temporary table using the given table
    create statement and populate statement. The population is done by
    partitioned xref table, and the statement will be formatted with dbid prior
    to being run.
    """

    cursor = connection.cursor()
    cursor.execute(table)
    cursor.execute('select id from rnc_database order by id')
    dbids = [r[0] for r in cursor.fetchall()]
    for dbid in dbids:
        cursor.execute(populate.format(dbid=dbid))

{'fraction_non_alphanumeric': 0.08547496668570341, 'fraction_numerical': 0.00799543118218161, 'mean_word_length': 2.7288857345635202, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4605466', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1401, 'n_words': 475}