<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import os
import glob

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

import solvebio


class Manifest(object):
    """
    Manifests aren't strictly resources, they represent a list
    of remote files (URLs) with additional information that
    can be used for validation (size and MD5).
    """
    manifest = None

    def __init__(self):
        self.manifest = {'files': []}

    def add_file(self, path):
        print("Uploading file: {0}".format(path))
        upload = solvebio.Upload.create(path=path)
        print("Successfuly uploaded file {0} (id:{1} size:{2} md5:{3})"
              .format(path, upload.id, upload.size, upload.md5))

        self.manifest['files'].append({
            'name': upload.filename,
            'format': upload.format,
            'size': upload.size,
            'md5': upload.md5,
            'base64_md5': upload.base64_md5,
            'url': upload.download_url()
        })

    def add_url(self, url, **kwargs):
        self.manifest['files'].append({
            'url': url,
            'name': kwargs.get('name'),
            'format': kwargs.get('format'),
            'size': kwargs.get('size'),
            'md5': kwargs.get('md5'),
            'base64_md5': kwargs.get('base64_md5')
        })

    def add(self, *args):
        """
        Add one or more files or URLs to the manifest.
        If files contains a glob, it is expanded.

        All files are uploaded to SolveBio. The Upload
        object is used to fill the manifest.
        """
        def _is_url(path):
            p = urlparse(path)
            return bool(p.scheme)

        for path in args:
            path = os.path.expandpath(path)

            if _is_url(path):
                self.add_url(path)
            elif os.path.isfile(path):
                self.add_file(path)
            elif os.path.isdir(path):
                for f in os.listdir(path):
                    self.add_file(f)
            elif glob.glob(path):
                for f in glob.glob(path):
                    self.add_file(f)
            else:
                raise ValueError(
                    'Manifest paths must be files, directories, or URLs. '
                    'The following extensions are supported: '
                    '.vcf .vcf.gz .json .json.gz')<commit_msg>expanduser not expandpath (fixes #105)<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import os
import glob

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

import solvebio


class Manifest(object):
    """
    Manifests aren't strictly resources, they represent a list
    of remote files (URLs) with additional information that
    can be used for validation (size and MD5).
    """
    manifest = None

    def __init__(self):
        self.manifest = {'files': []}

    def add_file(self, path):
        print("Uploading file: {0}".format(path))
        upload = solvebio.Upload.create(path=path)
        print("Successfuly uploaded file {0} (id:{1} size:{2} md5:{3})"
              .format(path, upload.id, upload.size, upload.md5))

        self.manifest['files'].append({
            'name': upload.filename,
            'format': upload.format,
            'size': upload.size,
            'md5': upload.md5,
            'base64_md5': upload.base64_md5,
            'url': upload.download_url()
        })

    def add_url(self, url, **kwargs):
        self.manifest['files'].append({
            'url': url,
            'name': kwargs.get('name'),
            'format': kwargs.get('format'),
            'size': kwargs.get('size'),
            'md5': kwargs.get('md5'),
            'base64_md5': kwargs.get('base64_md5')
        })

    def add(self, *args):
        """
        Add one or more files or URLs to the manifest.
        If files contains a glob, it is expanded.

        All files are uploaded to SolveBio. The Upload
        object is used to fill the manifest.
        """
        def _is_url(path):
            p = urlparse(path)
            return bool(p.scheme)

        for path in args:
            path = os.path.expanduser(path)
            if _is_url(path):
                self.add_url(path)
            elif os.path.isfile(path):
                self.add_file(path)
            elif os.path.isdir(path):
                for f in os.listdir(path):
                    self.add_file(f)
            elif glob.glob(path):
                for f in glob.glob(path):
                    self.add_file(f)
            else:
                raise ValueError(
                    'Manifest paths must be files, directories, or URLs. '
                    'The following extensions are supported: '
                    '.vcf .vcf.gz .json .json.gz')
{'fraction_non_alphanumeric': 0.11485668146951958, 'fraction_numerical': 0.010698425514735568, 'mean_word_length': 1.7774663677130045, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1658810', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1399, 'n_words': 397}