<commit_before>import os
import sys
import json
import logging
import argparse
import requests
import requests_toolbelt

from . import util
from . import tempdir as tempfile

logging.basicConfig(
    format='%(asctime)s %(levelname)8.8s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)

log = logging.getLogger()

logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library


OAUTH_TOKEN_VAR = '<PASSWORD>OAUTH_TOKEN'

with open(os.path.join(os.path.dirname(__file__), 'types.json')) as fd:
    type_map = json.load(fd)
KNOWN_FILETYPES = {ext: filetype for filetype, extensions in type_map.iteritems() for ext in extensions}


def guess_filetype(path):
    particles = os.path.basename(path).split('.')[1:]
    extentions = ['.' + '.'.join(particles[i:]) for i in range(len(particles))]
    for ext in extentions:
        filetype = KNOWN_FILETYPES.get(ext.lower())
        if filetype:
            break
    return filetype


def scan_folder(path):
    projects = []
    for dirpath, dirnames, filenames in os.walk(path):
        filenames = [fn for fn in filenames if not fn.startswith('.')]  # ignore dotfiles
        dirnames[:] = [dn for dn in dirnames if not dn.startswith('.')] # use slice assignment to influence walk
        relpath = os.path.relpath(dirpath, path)
        if relpath == '.':
            continue
        levels = relpath.split(os.sep)
        level_cnt = len(levels)
        files = []
        if level_cnt == 1:      # group
            if filenames:
                log.critical('Files not allowed at group level')
                sys.exit(1)
        elif level_cnt == 2:    # project
            sessions = []
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            project = {'group': levels[0], 'label': levels[1], 'sessions': sessions, 'files': files}
            projects.append(project)
        elif level_cnt == 3:    # subject
            if filenames:
                log.critical('Files not allowed at subject level')
                sys.exit(1)
        elif level_cnt == 4:    # session
            acquisitions = []
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            session = {'label': levels[3], 'subject': {'code': levels[2]}, 'acquisitions': acquisitions, 'files': files}
            sessions.append(session)
        elif level_cnt == 5:    # acquisition
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            packfiles = [{'path': os.path.join(dirpath, dn), 'type': dn} for dn in dirnames]
            acquisition = {'label': levels[4], 'files': files, 'packfiles': packfiles}
            acquisitions.append(acquisition)
        elif level_cnt == 6:    # packfile
            pass
        else:
            log.critical('Folder structure too deep')
            sys.exit(1)
        for f in files:
            f['type'] = guess_filetype(f['path'])
    return projects


def tweak_labels(projects):
    for p in projects:
        session_labels = [s['label'] for s in p['sessions']]
        if len(set(session_labels)) < len(session_labels):
            for s in p['sessions']:
                s['label'] = s['subject']['code'] + '_' + s['label']
    return projects


def print_upload_summary(projects):
    pass


def file_metadata(f, **kwargs):
    md = {'name': os.path.basename(f['path']), 'type': f['type']}
    md.update(kwargs)
    return md


def upload(projects, api_url, http_headers, http_params, insecure):
    with requests.Session() as rs:
        rs.verify = not insecure
        rs.headers = http_headers
        rs.params = http_params
        upload_url = api_url + '/upload/label'
        action_str = 'Upserting {}files to {}'
        file_str = '  {} {f[type]:10}: {f[path]}'

        for project in projects:
            group = project['group']
            p_label = group + ' > ' + project['label']<commit_msg>Omit type field from metadata if null<commit_after>import os
import sys
import json
import logging
import argparse
import requests
import requests_toolbelt

from . import util
from . import tempdir as tempfile

logging.basicConfig(
    format='%(asctime)s %(levelname)8.8s %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S',
)

log = logging.getLogger()

logging.getLogger('requests').setLevel(logging.WARNING) # silence Requests library


OAUTH_TOKEN_VAR = '<PASSWORD>'

with open(os.path.join(os.path.dirname(__file__), 'types.json')) as fd:
    type_map = json.load(fd)
KNOWN_FILETYPES = {ext: filetype for filetype, extensions in type_map.iteritems() for ext in extensions}


def guess_filetype(path):
    particles = os.path.basename(path).split('.')[1:]
    extentions = ['.' + '.'.join(particles[i:]) for i in range(len(particles))]
    for ext in extentions:
        filetype = KNOWN_FILETYPES.get(ext.lower())
        if filetype:
            break
    return filetype


def scan_folder(path):
    projects = []
    for dirpath, dirnames, filenames in os.walk(path):
        filenames = [fn for fn in filenames if not fn.startswith('.')]  # ignore dotfiles
        dirnames[:] = [dn for dn in dirnames if not dn.startswith('.')] # use slice assignment to influence walk
        relpath = os.path.relpath(dirpath, path)
        if relpath == '.':
            continue
        levels = relpath.split(os.sep)
        level_cnt = len(levels)
        files = []
        if level_cnt == 1:      # group
            if filenames:
                log.critical('Files not allowed at group level')
                sys.exit(1)
        elif level_cnt == 2:    # project
            sessions = []
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            project = {'group': levels[0], 'label': levels[1], 'sessions': sessions, 'files': files}
            projects.append(project)
        elif level_cnt == 3:    # subject
            if filenames:
                log.critical('Files not allowed at subject level')
                sys.exit(1)
        elif level_cnt == 4:    # session
            acquisitions = []
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            session = {'label': levels[3], 'subject': {'code': levels[2]}, 'acquisitions': acquisitions, 'files': files}
            sessions.append(session)
        elif level_cnt == 5:    # acquisition
            files = [{'path': os.path.join(dirpath, fn)} for fn in filenames]
            packfiles = [{'path': os.path.join(dirpath, dn), 'type': dn} for dn in dirnames]
            acquisition = {'label': levels[4], 'files': files, 'packfiles': packfiles}
            acquisitions.append(acquisition)
        elif level_cnt == 6:    # packfile
            pass
        else:
            log.critical('Folder structure too deep')
            sys.exit(1)
        for f in files:
            f['type'] = guess_filetype(f['path'])
    return projects


def tweak_labels(projects):
    for p in projects:
        session_labels = [s['label'] for s in p['sessions']]
        if len(set(session_labels)) < len(session_labels):
            for s in p['sessions']:
                s['label'] = s['subject']['code'] + '_' + s['label']
    return projects


def print_upload_summary(projects):
    pass


def file_metadata(f, **kwargs):
    md = {'name': os.path.basename(f['path'])}
    if f['type'] is not None:
        md['type'] = f['type']
    md.update(kwargs)
    return md


def upload(projects, api_url, http_headers, http_params, insecure):
    with requests.Session() as rs:
        rs.verify = not insecure
        rs.headers = http_headers
        rs.params = http_params
        upload_url = api_url + '/upload/label'
        action_str = 'Upserting {}files to {}'
        file_str = '  {} {f[type]:10}: {f[path]}'

        for project in projects:
            group = project['group']
            p_label = group + ' > ' + project['label']
{'fraction_non_alphanumeric': 0.13507680589056748, 'fraction_numerical': 0.004824171638948839, 'mean_word_length': 2.456779289161913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1520426', 'n_tokens_mistral': 2443, 'n_tokens_neox': 2401, 'n_words': 646}