<commit_before>from __future__ import print_function, absolute_import
import sys
import os
from os.path import exists, abspath, join as pth_join, dirname, isabs, relpath
from os import makedirs, mkdir, listdir, rename, unlink, stat

from contextlib import contextmanager
try:
    from os import scandir
except ImportError:
    from scandir import scandir as _scandir

    @contextmanager
    def scandir(directory):
        sd = _scandir(directory)
        try:
            yield sd
        finally:
            del sd

import hashlib
import shutil
import json
import logging
import errno
from collections import namedtuple
from six import string_types

try:
    from tempfile import TemporaryDirectory
except ImportError:
    from backports.tempfile import TemporaryDirectory

from .command_util import IVar, SubCommand
from .context import Context, DATA_CONTEXT_KEY, IMPORTS_CONTEXT_KEY
from .capability import provide, is_capable
from .capabilities import FilePathProvider
from .datasource_loader import DataSourceDirLoader, LoadFailed


L = logging.getLogger(__name__)

DEFAULT_SAVE_CALLABLE_NAME = 'pow_data'


class POWSourceData(object):
    ''' Commands for saving and loading data for DataSources '''
    def __init__(self, parent):
        self._source_command = parent
        self._pow_command = parent._parent

    def retrieve(self, source, archive='data.tar', archive_type=None):
        '''
        Retrieves the data for the source

        Parameters
        ----------
        source : str
            The source for data
        archive : str
            The file name of the archive. If this ends with an extension like
            '.zip', and no `archive_type` argument is given, then an archive
            will be created of that type. The archive name will *not* have any
            extension appended in any case.
        archive_type : str
            The type of the archive to create.
        '''
        sid = self._pow_command._den3(source)
        if not archive_type:
            for ext in EXT_TO_ARCHIVE_FMT:
                if archive.endswith(ext):
                    archive_type = EXT_TO_ARCHIVE_FMT.get(ext)
                    break

        if not archive_type:
            if ext:
                msg = "The extension '{}', does not match any known archive format." \
                        " Defaulting to TAR format"
                L.warning(msg.format(ext))
            archive_type = 'tar'

        try:
            dd = self._pow_command._dsd[sid]
        except KeyError:
            raise GenericUserError('Could not find data for {} ({})'.format(sid, source))

        with self._pow_command._tempdir(prefix='pow-source-data-retrieve.') as d:
            temp_archive = shutil.make_archive(pth_join(d, 'archive'), archive_type, dd)
            rename(temp_archive, archive)


EXT_TO_ARCHIVE_FMT = {
    '.tar.bz2': 'bztar',
    '.tar.gz': 'gztar',
    '.tar.xz': 'xztar',
    '.tar': 'tar',
    '.zip': 'zip',
}


class POWSource(object):
    ''' Commands for working with DataSource objects '''

    data = SubCommand(POWSourceData)

    def __init__(self, parent):
        self._parent = parent

    # Commands specific to sources
    def create(self, kind, key, *args, **kwargs):
        """
        Create the source and add it to the graph.

        Arguments are determined by the type of the data source

        Parameters
        ----------
        kind : rdflib.term.URIRef
            The kind of source to create
        key : str
            The key, a unique name for the source
        """

    def list(self, context=None, kind=None, full=False):
        """
        List known sources

        Parameters
        ----------
        kind : str
            Only list sources of this kind
        context : str
            The context to query for sources
        full : bool
            Whether to (attempt to) shorten the source URIs by using the namespace manager
        """
        from .datasource import DataSource
        conf = self._parent._conf()
        if context is not None:
            ctx = self._make_ctx(context)
        else:
            ctx = self._parent._data_ctx
        if kind is None:
            kind = DataSource.rdf_type
        kind_uri = self._parent._den3(kind)

        dst = ctx.stored(ctx.stored.resolve_class(kind_uri))
        dt = dst(conf=conf)
        nm = conf['rdf.graph'].namespace_manager
        for x in dt.load():
            if full:
                yield x.identifier
            else:
                yield nm.normalizeUri(x.identifier)

    def show(self, *data_source):
        '''
        Parameters
        ----------
        *data_source : str
            The ID of the data source to show
        '''
        from PyOpenWorm.datasource import DataSource

        for ds in data_source:
            uri = self._parent._den3(ds)
            for x in self._parent._data_ctx.stored(DataSource)(ident=uri).load():
                self._parent.message(x.format_str(stored=True))

    def list_kinds(self, full=False):
        """<commit_msg>Using query form for pow source list queries<commit_after>from __future__ import print_function, absolute_import
import sys
import os
from os.path import exists, abspath, join as pth_join, dirname, isabs, relpath
from os import makedirs, mkdir, listdir, rename, unlink, stat

from contextlib import contextmanager
try:
    from os import scandir
except ImportError:
    from scandir import scandir as _scandir

    @contextmanager
    def scandir(directory):
        sd = _scandir(directory)
        try:
            yield sd
        finally:
            del sd

import hashlib
import shutil
import json
import logging
import errno
from collections import namedtuple
from six import string_types

try:
    from tempfile import TemporaryDirectory
except ImportError:
    from backports.tempfile import TemporaryDirectory

from .command_util import IVar, SubCommand
from .context import Context, DATA_CONTEXT_KEY, IMPORTS_CONTEXT_KEY
from .capability import provide, is_capable
from .capabilities import FilePathProvider
from .datasource_loader import DataSourceDirLoader, LoadFailed


L = logging.getLogger(__name__)

DEFAULT_SAVE_CALLABLE_NAME = 'pow_data'


class POWSourceData(object):
    ''' Commands for saving and loading data for DataSources '''
    def __init__(self, parent):
        self._source_command = parent
        self._pow_command = parent._parent

    def retrieve(self, source, archive='data.tar', archive_type=None):
        '''
        Retrieves the data for the source

        Parameters
        ----------
        source : str
            The source for data
        archive : str
            The file name of the archive. If this ends with an extension like
            '.zip', and no `archive_type` argument is given, then an archive
            will be created of that type. The archive name will *not* have any
            extension appended in any case.
        archive_type : str
            The type of the archive to create.
        '''
        sid = self._pow_command._den3(source)
        if not archive_type:
            for ext in EXT_TO_ARCHIVE_FMT:
                if archive.endswith(ext):
                    archive_type = EXT_TO_ARCHIVE_FMT.get(ext)
                    break

        if not archive_type:
            if ext:
                msg = "The extension '{}', does not match any known archive format." \
                        " Defaulting to TAR format"
                L.warning(msg.format(ext))
            archive_type = 'tar'

        try:
            dd = self._pow_command._dsd[sid]
        except KeyError:
            raise GenericUserError('Could not find data for {} ({})'.format(sid, source))

        with self._pow_command._tempdir(prefix='pow-source-data-retrieve.') as d:
            temp_archive = shutil.make_archive(pth_join(d, 'archive'), archive_type, dd)
            rename(temp_archive, archive)


EXT_TO_ARCHIVE_FMT = {
    '.tar.bz2': 'bztar',
    '.tar.gz': 'gztar',
    '.tar.xz': 'xztar',
    '.tar': 'tar',
    '.zip': 'zip',
}


class POWSource(object):
    ''' Commands for working with DataSource objects '''

    data = SubCommand(POWSourceData)

    def __init__(self, parent):
        self._parent = parent

    # Commands specific to sources
    def create(self, kind, key, *args, **kwargs):
        """
        Create the source and add it to the graph.

        Arguments are determined by the type of the data source

        Parameters
        ----------
        kind : rdflib.term.URIRef
            The kind of source to create
        key : str
            The key, a unique name for the source
        """

    def list(self, context=None, kind=None, full=False):
        """
        List known sources

        Parameters
        ----------
        kind : str
            Only list sources of this kind
        context : str
            The context to query for sources
        full : bool
            Whether to (attempt to) shorten the source URIs by using the namespace manager
        """
        from .datasource import DataSource
        conf = self._parent._conf()
        if context is not None:
            ctx = self._make_ctx(context)
        else:
            ctx = self._parent._data_ctx
        if kind is None:
            kind = DataSource.rdf_type
        kind_uri = self._parent._den3(kind)

        dst = ctx.stored(ctx.stored.resolve_class(kind_uri))
        dt = dst.query(conf=conf)
        nm = conf['rdf.graph'].namespace_manager
        for x in dt.load():
            if full:
                yield x.identifier
            else:
                yield nm.normalizeUri(x.identifier)

    def show(self, *data_source):
        '''
        Parameters
        ----------
        *data_source : str
            The ID of the data source to show
        '''
        from PyOpenWorm.datasource import DataSource

        for ds in data_source:
            uri = self._parent._den3(ds)
            for x in self._parent._data_ctx.stored(DataSource)(ident=uri).load():
                self._parent.message(x.format_str(stored=True))

    def list_kinds(self, full=False):
        """
{'fraction_non_alphanumeric': 0.08376808741017816, 'fraction_numerical': 0.0007874790825868688, 'mean_word_length': 2.04282719377059, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2057910', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2735, 'n_words': 905}