<commit_before>"""
A supercomputer plugin, specificially for
`Cedar <https://docs.computecanada.ca/wiki/Cedar>`_.
"""
import os
import sys
import stat
import logging
import getpass
from datetime import datetime,timedelta
import subprocess
import asyncio
from functools import partial

from .condor_direct import check_call, check_output, MyServerComms, TaskInfo, run_async, condor_direct

logger = logging.getLogger('plugin-cedar')

async def subprocess_ssh(host, args):
    """A subprocess call over ssh"""
    cmd = ['ssh', '-i', '~/.ssh/iceprod', host]
    if 'ICEPRODBASE' in os.environ:
        cmd.append(f'{os.environ["ICEPRODBASE"]}/env-shell.sh')
    cmd += args
    logger.info('subprocess_ssh: %r', cmd)
    p = await asyncio.create_subprocess_exec(*cmd)
    await asyncio.wait_for(p.wait(), timeout=20*60)
    if p.returncode:
        raise Exception(f'command failed, return code {p.returncode}')
    return p


class supercomp_cedar(condor_direct):

    ### Plugin Overrides ###

    batch_site = 'Cedar'
    batch_outfile = 'slurm.out'

    # Cedar queue requirements
    batch_resources = {
        'cpu': 1,
        'memory': 3.75,
        'disk': 48.,
        'time': 20.,
        'os': 'RHEL_7_x86_64',
    }
<commit_msg>need all default types in here<commit_after>"""
A supercomputer plugin, specificially for
`Cedar <https://docs.computecanada.ca/wiki/Cedar>`_.
"""
import os
import sys
import stat
import logging
import getpass
from datetime import datetime,timedelta
import subprocess
import asyncio
from functools import partial

from .condor_direct import check_call, check_output, MyServerComms, TaskInfo, run_async, condor_direct

logger = logging.getLogger('plugin-cedar')

async def subprocess_ssh(host, args):
    """A subprocess call over ssh"""
    cmd = ['ssh', '-i', '~/.ssh/iceprod', host]
    if 'ICEPRODBASE' in os.environ:
        cmd.append(f'{os.environ["ICEPRODBASE"]}/env-shell.sh')
    cmd += args
    logger.info('subprocess_ssh: %r', cmd)
    p = await asyncio.create_subprocess_exec(*cmd)
    await asyncio.wait_for(p.wait(), timeout=20*60)
    if p.returncode:
        raise Exception(f'command failed, return code {p.returncode}')
    return p


class supercomp_cedar(condor_direct):

    ### Plugin Overrides ###

    batch_site = 'Cedar'
    batch_outfile = 'slurm.out'

    # Cedar queue requirements
    batch_resources = {
        'cpu': 1,
        'gpu': 0,
        'memory': 3.75,
        'disk': 48.,
        'time': 20.,
        'os': 'RHEL_7_x86_64',
    }

{'fraction_non_alphanumeric': 0.12962226640159047, 'fraction_numerical': 0.013916500994035786, 'mean_word_length': 3.5251798561151078, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6947973', 'n_tokens_mistral': 886, 'n_tokens_neox': 842, 'n_words': 213}