<commit_before>"""Download-related logic"""

import aiofiles
import aiohttp
import os

from .log import download_logger

# TODO: Move to a separate service

error_patterns = (
    b'Error reading from remote server',
    b'Bad gateway',
    b'Cannot process ID list',
    b'server is temporarily unable to service your request',
    b'Service unavailable',
    b'Server Error',
    b'ID list is empty',
    b'Resource temporarily unavailable',
)


async def download(job, app):
    conf = app['run_conf']
    async with aiohttp.ClientSession(loop=app.loop) as session:

        params = {
            'tool': 'antiSMASH downloader',
            'retmode': 'text',
            'id': job.download,
        }

        if job.molecule_type == 'nucl':
            params['db'] = 'nucleotide'
            params['rettype'] = 'gbwithparts'
            file_ending = '.gbk'
        elif job.molecule_type == 'prot':
            params['db'] = 'protein'
            params['rettype'] = 'fasta'
            file_ending = '.fa'
        else:
            download_logger.error("Invalid molecule_type %r, ignoring download.", job.molecule_type)
            job.state = 'failed'
            job.status = 'failed: Invalid molecule type {}'.format(job.molecule_type)
            await job.commit()
            return False

        outdir = os.path.join(conf.workdir, job.job_id)
        os.makedirs(outdir, exist_ok=True)

        base_filename = job.download + file_ending
        outfile = os.path.join(outdir, base_filename)
        job.filename = base_filename

        job.state = 'downloading'
        job.status = 'downloading: Getting {} from NCBI'.format(job.download)
        await job.commit()

        async with session.get(conf.entrez_url, params=params) as response, aiofiles.open(outfile, 'wb') as fh:
            while True:
                chunk = await response.content.read(4096)
                if not chunk:
                    job.state = 'running'
                    job.status = 'running: Downloaded {}'.format(base_filename)
                    break
                for pattern in error_patterns:
                    if pattern in chunk:
                        job.state = 'failed'
                        job.status = "Failed to download file with id {} from NCBI: {}".format(job.download, pattern)
                        break
                await fh.write(chunk)

        await job.commit()
        return True
<commit_msg>download: One more error pattern<commit_after>"""Download-related logic"""

import aiofiles
import aiohttp
import os

from .log import download_logger

# TODO: Move to a separate service

error_patterns = (
    b'Error reading from remote server',
    b'Bad gateway',
    b'Cannot process ID list',
    b'server is temporarily unable to service your request',
    b'Service unavailable',
    b'Server Error',
    b'ID list is empty',
    b'Resource temporarily unavailable',
    b'In it there are neither IDs nor accessions',
)


async def download(job, app):
    conf = app['run_conf']
    async with aiohttp.ClientSession(loop=app.loop) as session:

        params = {
            'tool': 'antiSMASH downloader',
            'retmode': 'text',
            'id': job.download,
        }

        if job.molecule_type == 'nucl':
            params['db'] = 'nucleotide'
            params['rettype'] = 'gbwithparts'
            file_ending = '.gbk'
        elif job.molecule_type == 'prot':
            params['db'] = 'protein'
            params['rettype'] = 'fasta'
            file_ending = '.fa'
        else:
            download_logger.error("Invalid molecule_type %r, ignoring download.", job.molecule_type)
            job.state = 'failed'
            job.status = 'failed: Invalid molecule type {}'.format(job.molecule_type)
            await job.commit()
            return False

        outdir = os.path.join(conf.workdir, job.job_id)
        os.makedirs(outdir, exist_ok=True)

        base_filename = job.download + file_ending
        outfile = os.path.join(outdir, base_filename)
        job.filename = base_filename

        job.state = 'downloading'
        job.status = 'downloading: Getting {} from NCBI'.format(job.download)
        await job.commit()

        async with session.get(conf.entrez_url, params=params) as response, aiofiles.open(outfile, 'wb') as fh:
            while True:
                chunk = await response.content.read(4096)
                if not chunk:
                    job.state = 'running'
                    job.status = 'running: Downloaded {}'.format(base_filename)
                    break
                for pattern in error_patterns:
                    if pattern in chunk:
                        job.state = 'failed'
                        job.status = "Failed to download file with id {} from NCBI: {}".format(job.download, pattern)
                        break
                await fh.write(chunk)

        await job.commit()
        return True

{'fraction_non_alphanumeric': 0.10375454178441663, 'fraction_numerical': 0.0016148566814695195, 'mean_word_length': 1.9406528189910979, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475859', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1343, 'n_words': 372}