<commit_before>#!/usr/bin/env python2.7

# ------------------------------------------------------------------------------
# Script which imports portal data.
#
# ------------------------------------------------------------------------------

import os
import sys
import argparse
import re

import cbioportal_common
from cbioportal_common import OUTPUT_FILE
from cbioportal_common import ERROR_FILE
from cbioportal_common import MetaFileTypes
from cbioportal_common import IMPORTER_CLASSNAME_BY_META_TYPE
from cbioportal_common import IMPORTER_REQUIRES_METADATA
from cbioportal_common import IMPORT_CANCER_TYPE_CLASS
from cbioportal_common import IMPORT_STUDY_CLASS
from cbioportal_common import REMOVE_STUDY_CLASS
from cbioportal_common import IMPORT_CASE_LIST_CLASS
from cbioportal_common import get_metastudy_properties
from cbioportal_common import get_metafile_properties
from cbioportal_common import get_properties
from cbioportal_common import run_java


# ------------------------------------------------------------------------------
# globals

# commands
IMPORT_CANCER_TYPE = "import-cancer-type"
IMPORT_STUDY = "import-study"
REMOVE_STUDY = "remove-study"
IMPORT_STUDY_DATA = "import-study-data"
IMPORT_CASE_LIST = "import-case-list"

COMMANDS = [IMPORT_CANCER_TYPE, IMPORT_STUDY, REMOVE_STUDY, IMPORT_STUDY_DATA, IMPORT_CASE_LIST]


# ------------------------------------------------------------------------------
# sub-routines

def import_cancer_type(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_CANCER_TYPE_CLASS)
    args.append(meta_filename)
    args.append("false") # don't clobber existing table
    run_java(*args)

def import_study(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_STUDY_CLASS)
    args.append(meta_filename)
    run_java(*args)

def remove_study(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(REMOVE_STUDY_CLASS)
    metastudy_properties = get_metastudy_properties(meta_filename)
    args.append(metastudy_properties.cancer_study_identifier)
    run_java(*args)

def import_study_data(jvm_args, meta_filename, data_filename):
    args = jvm_args.split(' ')
    metafile_properties = get_metafile_properties(meta_filename)

    meta_file_type = cbioportal_common.get_meta_file_type(metafile_properties)
    if meta_file_type is None:
        print >> ERROR_FILE, ("Could not determine meta file type, consider "
                              "running the validator script.")
        return
    importer = IMPORTER_CLASSNAME_BY_META_TYPE[metafile_properties.meta_file_type]

    args.append(importer)
    if IMPORTER_REQUIRES_METADATA[importer]:
        args.append("--meta")
        args.append(meta_filename)
        args.append("--loadMode")
        args.append("bulkload")
    if meta_file_type == MetaFileTypes.CLINICAL:
        args.append(data_filename)
        args.append(metafile_properties.cancer_study_identifier)
    else:
        args.append("--data")
        args.append(data_filename)
    run_java(*args)

def import_case_list(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_CASE_LIST_CLASS)
    args.append(meta_filename)
    run_java(*args)

def process_case_lists(jvm_args, case_list_dir):
    case_list_files = (os.path.join(case_list_dir, x) for
                       x in os.listdir(case_list_dir))
    for case_list in case_list_files:
        import_case_list(jvm_args,case_list)

def process_command(jvm_args, command, meta_filename, data_filename):
    if command == IMPORT_CANCER_TYPE:
        import_cancer_type(jvm_args, meta_filename)
    elif command == IMPORT_STUDY:
        import_study(jvm_args, meta_filename)
    elif command == REMOVE_STUDY:
        remove_study(jvm_args, meta_filename)
    elif command == IMPORT_STUDY_DATA:
        import_study_data(jvm_args, meta_filename, data_filename)
    elif command == IMPORT_CASE_LIST:
        import_case_list(jvm_args, meta_filename)

def process_directory(jvm_args, study_directory):
    meta_filenames = (
        os.path.join(study_directory, f) for
        f in os.listdir(study_directory) if
        re.search(r'(\b|_)meta(\b|_)', f) and
        not (f.startswith('.') or f.endswith('~')))
    study_meta = {}
    clinical_metafiles = []
    non_clinical_metafiles = []
    cancer_type_meta = {}

    for f in meta_filenames:
        metadata = get_properties(f)
        meta_file_type = cbioportal_common.get_meta_file_type(metadata)
        if meta_file_type is None:
            print >> ERROR_FILE, (
                    "Could not determine meta file type for {}, consider "
                    "running the validator script.".format(f))
            return
        if meta_file_type == MetaFileTypes.STUDY:
            study_meta = metadata
            #First remove study if exists
            remove_study(jvm_args,f)
            #Then import study
            import_study(jvm_args,f)
        elif meta_file_type == MetaFileTypes.CANCER_TYPE:
            # TODO this is a bug, make it a .append()
            cancer_type_meta = metadata
        elif meta_file_type == MetaFileTypes.CLINICAL:
            clinical_metafiles.append(f)
        else:
            non_clinical_metafiles.append(f)

    if len(study_meta) == 0:
        print >> ERROR_FILE, 'No meta_study file found'
        sys.exit(1)

    # First, import cancer type
    if cancer_type_meta != {}:
        import_cancer_type(jvm_args, f)

    # Next, we need to import clinical files
    for f in clinical_metafiles:
        metadata = get_properties(f)
        import_study_data(jvm_args, f,
                          os.path.join(study_directory,<commit_msg>Allow importing multiple cancer type files in one study<commit_after>#!/usr/bin/env python2.7

# ------------------------------------------------------------------------------
# Script which imports portal data.
#
# ------------------------------------------------------------------------------

import os
import sys
import argparse
import re

import cbioportal_common
from cbioportal_common import OUTPUT_FILE
from cbioportal_common import ERROR_FILE
from cbioportal_common import MetaFileTypes
from cbioportal_common import IMPORTER_CLASSNAME_BY_META_TYPE
from cbioportal_common import IMPORTER_REQUIRES_METADATA
from cbioportal_common import IMPORT_CANCER_TYPE_CLASS
from cbioportal_common import IMPORT_STUDY_CLASS
from cbioportal_common import REMOVE_STUDY_CLASS
from cbioportal_common import IMPORT_CASE_LIST_CLASS
from cbioportal_common import get_metastudy_properties
from cbioportal_common import get_metafile_properties
from cbioportal_common import get_properties
from cbioportal_common import run_java


# ------------------------------------------------------------------------------
# globals

# commands
IMPORT_CANCER_TYPE = "import-cancer-type"
IMPORT_STUDY = "import-study"
REMOVE_STUDY = "remove-study"
IMPORT_STUDY_DATA = "import-study-data"
IMPORT_CASE_LIST = "import-case-list"

COMMANDS = [IMPORT_CANCER_TYPE, IMPORT_STUDY, REMOVE_STUDY, IMPORT_STUDY_DATA, IMPORT_CASE_LIST]


# ------------------------------------------------------------------------------
# sub-routines

def import_cancer_type(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_CANCER_TYPE_CLASS)
    args.append(meta_filename)
    args.append("false") # don't clobber existing table
    run_java(*args)

def import_study(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_STUDY_CLASS)
    args.append(meta_filename)
    run_java(*args)

def remove_study(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(REMOVE_STUDY_CLASS)
    metastudy_properties = get_metastudy_properties(meta_filename)
    args.append(metastudy_properties.cancer_study_identifier)
    run_java(*args)

def import_study_data(jvm_args, meta_filename, data_filename):
    args = jvm_args.split(' ')
    metafile_properties = get_metafile_properties(meta_filename)

    meta_file_type = cbioportal_common.get_meta_file_type(metafile_properties)
    if meta_file_type is None:
        print >> ERROR_FILE, ("Could not determine meta file type, consider "
                              "running the validator script.")
        return
    importer = IMPORTER_CLASSNAME_BY_META_TYPE[metafile_properties.meta_file_type]

    args.append(importer)
    if IMPORTER_REQUIRES_METADATA[importer]:
        args.append("--meta")
        args.append(meta_filename)
        args.append("--loadMode")
        args.append("bulkload")
    if meta_file_type == MetaFileTypes.CLINICAL:
        args.append(data_filename)
        args.append(metafile_properties.cancer_study_identifier)
    else:
        args.append("--data")
        args.append(data_filename)
    run_java(*args)

def import_case_list(jvm_args, meta_filename):
    args = jvm_args.split(' ')
    args.append(IMPORT_CASE_LIST_CLASS)
    args.append(meta_filename)
    run_java(*args)

def process_case_lists(jvm_args, case_list_dir):
    case_list_files = (os.path.join(case_list_dir, x) for
                       x in os.listdir(case_list_dir))
    for case_list in case_list_files:
        import_case_list(jvm_args,case_list)

def process_command(jvm_args, command, meta_filename, data_filename):
    if command == IMPORT_CANCER_TYPE:
        import_cancer_type(jvm_args, meta_filename)
    elif command == IMPORT_STUDY:
        import_study(jvm_args, meta_filename)
    elif command == REMOVE_STUDY:
        remove_study(jvm_args, meta_filename)
    elif command == IMPORT_STUDY_DATA:
        import_study_data(jvm_args, meta_filename, data_filename)
    elif command == IMPORT_CASE_LIST:
        import_case_list(jvm_args, meta_filename)

def process_directory(jvm_args, study_directory):
    meta_filenames = (
        os.path.join(study_directory, f) for
        f in os.listdir(study_directory) if
        re.search(r'(\b|_)meta(\b|_)', f) and
        not (f.startswith('.') or f.endswith('~')))
    study_meta = {}
    clinical_metafiles = []
    non_clinical_metafiles = []
    cancer_type_metafiles = []

    for f in meta_filenames:
        metadata = get_properties(f)
        meta_file_type = cbioportal_common.get_meta_file_type(metadata)
        if meta_file_type is None:
            print >> ERROR_FILE, (
                    "Could not determine meta file type for {}, consider "
                    "running the validator script.".format(f))
            return
        if meta_file_type == MetaFileTypes.STUDY:
            study_meta = metadata
            #First remove study if exists
            remove_study(jvm_args,f)
            #Then import study
            import_study(jvm_args,f)
        elif meta_file_type == MetaFileTypes.CANCER_TYPE:
            cancer_type_metafiles.append(f)
        elif meta_file_type == MetaFileTypes.CLINICAL:
            clinical_metafiles.append(f)
        else:
            non_clinical_metafiles.append(f)

    if len(study_meta) == 0:
        print >> ERROR_FILE, 'No meta_study file found'
        sys.exit(1)

    # First, import cancer types
    for f in cancer_type_metafiles:
        import_cancer_type(jvm_args, f)

    # Next, we need to import clinical files
    for f in clinical_metafiles:
        metadata = get_properties(f)
        import_study_data(jvm_args, f,
                          os.path.join(study_directory,
{'fraction_non_alphanumeric': 0.12991468027091213, 'fraction_numerical': 0.0007036678687659425, 'mean_word_length': 2.7912637545848615, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7584594', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3527, 'n_words': 673}