<commit_before>r"""Qitta study handlers for the Tornado webserver.
"""
# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from __future__ import division
from collections import namedtuple

from tornado.web import authenticated, HTTPError, asynchronous
from tornado.gen import coroutine, Task
from wtforms import (Form, StringField, SelectField, BooleanField,
                     SelectMultipleField, TextAreaField, validators)

from future.utils import viewitems

from operator import itemgetter

from traceback import format_exception_only
from sys import exc_info

from json import dumps
from os import listdir, remove
from os.path import exists, join, basename
from functools import partial
from .base_handlers import BaseHandler

from pandas.parser import CParserError

from qiita_core.exceptions import IncompetentQiitaDeveloperError
from qiita_pet.util import linkify
from qiita_ware.context import submit
from qiita_ware.util import dataframe_from_template, stats_from_df
from qiita_ware.demux import stats as demux_stats
from qiita_ware.dispatchable import submit_to_ebi
from qiita_db.metadata_template import (SampleTemplate, PrepTemplate,
                                        load_template_to_dataframe)
from qiita_db.study import Study, StudyPerson
from qiita_db.user import User
from qiita_db.util import (get_study_fp, get_filepath_types, get_data_types,
                           get_filetypes, convert_to_id)
from qiita_db.data import PreprocessedData, RawData
from qiita_db.exceptions import (QiitaDBColumnError, QiitaDBExecutionError,
                                 QiitaDBDuplicateError, QiitaDBUnknownIDError)
from qiita_db.ontology import Ontology

study_person_linkifier = partial(
    linkify, "<a target=\"_blank\" href=\"mailto:{0}\">{1}</a>")
pubmed_linkifier = partial(
    linkify, "<a target=\"_blank\" href=\"http://www.ncbi.nlm.nih.gov/"
    "pubmed/{0}\">{0}</a>")


def _get_shared_links_for_study(study):
    shared = []
    for person in study.shared_with:
        person = User(person)
        shared.append(study_person_linkifier(
            (person.email, person.info['name'])))
    return ", ".join(shared)


def _build_study_info(studytype, user=None):
        """builds list of namedtuples for study listings"""
        if studytype == "private":
            studylist = user.private_studies
        elif studytype == "shared":
            studylist = user.shared_studies
        elif studytype == "public":
            studylist = Study.get_public()
        else:
            raise IncompetentQiitaDeveloperError("Must use private, shared, "
                                                 "or public!")

        StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
                                'num_samples_collected shared num_raw_data pi '
                                'pmids owner status')

        infolist = []
        for s_id in studylist:
            study = Study(s_id)
            status = study.status
            # Just passing the email address as the name here, since
            # name is not a required field in qiita.qiita_user
            owner = study_person_linkifier((study.owner, study.owner))
            info = study.info
            PI = StudyPerson(info['principal_investigator_id'])
            PI = study_person_linkifier((PI.email, PI.name))
            pmids = ", ".join([pubmed_linkifier([pmid])
                               for pmid in study.pmids])
            shared = _get_shared_links_for_study(study)
            infolist.append(StudyTuple(study.id, study.title,
                                       info["metadata_complete"],
                                       info["number_samples_collected"],
                                       shared, len(study.raw_data()),
                                       PI, pmids, owner, status))
        return infolist


def check_access(user, study, no_public=False):
    """make sure user has access to the study requested"""
    if not study.has_access(user, no_public):
        if no_public:
            return False
        else:
            raise HTTPError(403, "User %s does not have access to study %d" %
                                 (user.id, study.id))
    return True


def _check_owner(user, study):
    """make sure user is the owner of the study requested"""
    if not user == study.owner:
        raise HTTPError(403, "User %s does not own study %d" %
                        (user, study.id))


class CreateStudyForm(Form):
    study_title = StringField('Study Title', [validators.required()])
    study_alias = StringField('Study Alias', [validators.required()])
    pubmed_id = StringField('PubMed ID')

    # TODO:This can be filled from the database
    # in oracle, this is in controlled_vocabs (ID 1),
    #                       controlled_vocab_values with CVV IDs >= 0
    environmental_packages = SelectMultipleField(
        'Environmental Packages',
        [validators.required()],
        choices=[('air', 'air'),
                 ('host_associated', 'host-associated'),
                 ('human_amniotic_fluid', 'human-amniotic-fluid'),
                 ('human_associated', 'human-associated'),
                 ('human_blood', 'human-blood'),
                 ('human_gut', 'human-gut'),
                 ('human_oral', 'human-oral'),
                 ('human_skin', 'human-skin'),
                 ('human_urine', 'human-urine'),
                 ('human_vaginal', 'human-vaginal'),
                 ('biofilm', 'microbial mat/biofilm'),
                 ('misc_env',
                  'miscellaneous natural or artificial environment'),
                 ('plant_associated', 'plant-associated'),
                 ('sediment', 'sediment'),
                 ('soil', 'soil'),
                 ('wastewater_sludge', 'wastewater/sludge'),
                 ('water', 'water')])
    is_timeseries = BooleanField('Includes Event-Based Data')
    study_abstract = TextAreaField('Study Abstract', [validators.required()])
    study_description = StringField('Study Description',
                                    [validators.required()])
    # The choices for these "people" fields will be filled from the database
    principal_investigator = SelectField('Principal Investigator',
                                         [validators.required()],
                                         coerce=lambda x: x)
    lab_person = SelectField('Lab Person', coerce=lambda x: x)


class PrivateStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        user = User(self.current_user)
        user_studies = yield Task(self._get_private, user)
        shared_studies = yield Task(self._get_shared, user)
        all_emails_except_current = yield Task(self._get_all_emails)
        all_emails_except_current.remove(self.current_user)
        self.render('private_studies.html', user=self.current_user,
                    user_studies=user_studies, shared_studies=shared_studies,
                    all_emails_except_current=all_emails_except_current)

    def _get_private(self, user, callback):
        callback(_build_study_info("private", user))

    def _get_shared(self, user, callback):
        """builds list of tuples for studies that are shared with user"""
        callback(_build_study_info("shared", user))

    def _get_all_emails(self, callback):
        callback(list(User.iter()))


class PublicStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        public_studies = yield Task(self._get_public)
        self.render('public_studies.html', user=self.current_user,
                    public_studies=public_studies)

    def _get_public(self, callback):
        """builds list of tuples for studies that are public"""
        callback(_build_study_info("public"))


class StudyDescriptionHandler(BaseHandler):
    def get_raw_data(self, rdis, callback):
        """Get all raw data objects from a list of raw_data_ids"""
        callback([RawData(rdi) for rdi in rdis])

    def get_prep_templates(self, raw_data, callback):
        """Get all prep templates for a list of raw data objects"""
        d = {}
        for rd in raw_data:
            # We neeed this so PrepTemplate(p) doesn't fail if that raw
            # doesn't exist but raw data has the row: #554
            prep_templates = sorted(rd.prep_templates)
            d[rd.id] = [PrepTemplate(p) for p in prep_templates
                        if PrepTemplate.exists(p)]
        callback(d)

    def remove_add_study_template(self, raw_data, study_id, fp_rsp, callback):
        """Removes prep templates, raw data and sample template and adds
           a new one
        """
        for rd in raw_data():
            if PrepTemplate.exists((rd)):
                PrepTemplate.delete(rd)
        if SampleTemplate.exists(study_id):
            SampleTemplate.delete(study_id)

        SampleTemplate.create(load_template_to_dataframe(fp_rsp),
                              Study(study_id))
        # TODO: do not remove but move to final storage space
        # and keep there forever, issue #550
        remove(fp_rsp)

        callback()

    def get_raw_data_from_other_studies(self, user, study, callback):
        """Retrieves a tuple of raw_data_id and the last study title for that
        raw_data
        """
        d = {}
        for sid in user.private_studies:
            if sid == study.id:
                continue
            for rdid in Study(sid).raw_data():
                d[rdid] = Study(RawData(rdid).studies[-1]).title
        callback(d)

    @coroutine
    def display_template(self, study_id, msg, msg_level, tab_to_display=""):
        """Simple function to avoid duplication of code"""
        # make sure study is accessible and exists, raise error if not
        study = None
        study_id = int(study_id)
        try:
            study = Study(study_id)
        except QiitaDBUnknownIDError:
            # Study not in database so fail nicely
            raise HTTPError(404, "Study %d does not exist" % study_id)
        else:
            check_access(User(self.current_user), study)

        # processing files from upload path
        fp = get_study_fp(study_id)
        if exists(fp):
            fs = listdir(fp)
        else:
            fs = []
        # getting raw filepath_ types
        fts = [k.split('_', 1)[1].replace('_', ' ')
               for k in get_filepath_types() if k.startswith('raw_')]
        fts = ['<option value="%s">%s</option>' % (f, f) for f in fts]

        study = Study(study_id)
        user = User(self.current_user)
        # getting the RawData and its prep templates
        available_raw_data = yield Task(self.get_raw_data, study.raw_data())
        available_prep_templates = yield Task(self.get_prep_templates,
                                              available_raw_data)

        # other general vars, note that we create the select options here
        # so we do not have to loop several times over them in the template
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
        data_types = ['<option value="%s">%s</option>' % (v, k)
                      for k, v in data_types]
        filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1))
        filetypes = ['<option value="%s">%s</option>' % (v, k)
                     for k, v in filetypes]
        other_studies_rd = yield Task(self.get_raw_data_from_other_studies,
                                      user, study)
        other_studies_rd = ['<option value="%s">%s</option>' % (k,
                            "id: %d, study: %s" % (k, v))
                            for k, v in viewitems(other_studies_rd)]

        ontology = Ontology(convert_to_id('ENA', 'ontology'))

        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        self.render('study_description.html', user=self.current_user,
                    study_title=study.title, study_info=study.info,
                    study_id=study_id, files=fs, filetypes=''.join(filetypes),
                    user_level=user.level, data_types=''.join(data_types),
                    available_raw_data=available_raw_data,
                    available_prep_templates=available_prep_templates,
                    ste=SampleTemplate.exists(study_id),
                    filepath_types=''.join(fts), ena_terms=''.join(ena_terms),
                    tab_to_display=tab_to_display,
                    level=msg_level, message=msg,
                    can_upload=check_access(user, study, True),
                    other_studies_rd=''.join(other_studies_rd),
                    user_defined_terms=user_defined_terms)

    @authenticated
    def get(self, study_id):
        study_id = int(study_id)
        check_access(User(self.current_user), Study(study_id))
        self.display_template(int(study_id), "", "")

    @authenticated
    @coroutine
    def post(self, study_id):
        study_id = int(study_id)
        check_access(User(self.current_user), Study(study_id))

        # vars to add sample template
        sample_template = self.get_argument('sample_template', None)
        # vars to add raw data
        filetype = self.get_argument('filetype', None)
        previous_raw_data = self.get_argument('previous_raw_data', None)
        # vars to add prep template
        add_prep_template = self.get_argument('add_prep_template', None)
        raw_data_id = self.get_argument('raw_data_id', None)
        data_type_id = self.get_argument('data_type_id', None)
        investigation_type = self.get_argument('investigation-type', None)
        user_defined_investigation_type = self.get_argument(
            'user-defined-investigation-type', None)
        new_investigation_type = self.get_argument('new-investigation-type',
                                                   None)
        if investigation_type == "":
            investigation_type = None

        # to update investigation type
        update_investigation_type = self.get_argument(
            'update_investigation_type', None)
        edit_investigation_type = self.get_argument('edit-investigation-type',
                                                    None)
        edit_user_defined_investigation_type = self.get_argument(
            'edit-user-defined-investigation-type', None)
        edit_new_investigation_type = self.get_argument(
            'edit-new-investigation-type', None)

        study = Study(study_id)
        msg_level = 'success'
        if sample_template:
            # processing sample templates

            fp_rsp = join(get_study_fp(study_id), sample_template)
            if not exists(fp_rsp):
                raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
<commit_msg>ENH: Make investigation_type truly optional<commit_after>r"""Qitta study handlers for the Tornado webserver.
"""
# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from __future__ import division
from collections import namedtuple

from tornado.web import authenticated, HTTPError, asynchronous
from tornado.gen import coroutine, Task
from wtforms import (Form, StringField, SelectField, BooleanField,
                     SelectMultipleField, TextAreaField, validators)

from future.utils import viewitems

from operator import itemgetter

from traceback import format_exception_only
from sys import exc_info

from json import dumps
from os import listdir, remove
from os.path import exists, join, basename
from functools import partial
from .base_handlers import BaseHandler

from pandas.parser import CParserError

from qiita_core.exceptions import IncompetentQiitaDeveloperError
from qiita_pet.util import linkify
from qiita_ware.context import submit
from qiita_ware.util import dataframe_from_template, stats_from_df
from qiita_ware.demux import stats as demux_stats
from qiita_ware.dispatchable import submit_to_ebi
from qiita_db.metadata_template import (SampleTemplate, PrepTemplate,
                                        load_template_to_dataframe)
from qiita_db.study import Study, StudyPerson
from qiita_db.user import User
from qiita_db.util import (get_study_fp, get_filepath_types, get_data_types,
                           get_filetypes, convert_to_id)
from qiita_db.data import PreprocessedData, RawData
from qiita_db.exceptions import (QiitaDBColumnError, QiitaDBExecutionError,
                                 QiitaDBDuplicateError, QiitaDBUnknownIDError)
from qiita_db.ontology import Ontology

study_person_linkifier = partial(
    linkify, "<a target=\"_blank\" href=\"mailto:{0}\">{1}</a>")
pubmed_linkifier = partial(
    linkify, "<a target=\"_blank\" href=\"http://www.ncbi.nlm.nih.gov/"
    "pubmed/{0}\">{0}</a>")


def _get_shared_links_for_study(study):
    shared = []
    for person in study.shared_with:
        person = User(person)
        shared.append(study_person_linkifier(
            (person.email, person.info['name'])))
    return ", ".join(shared)


def _build_study_info(studytype, user=None):
        """builds list of namedtuples for study listings"""
        if studytype == "private":
            studylist = user.private_studies
        elif studytype == "shared":
            studylist = user.shared_studies
        elif studytype == "public":
            studylist = Study.get_public()
        else:
            raise IncompetentQiitaDeveloperError("Must use private, shared, "
                                                 "or public!")

        StudyTuple = namedtuple('StudyInfo', 'id title meta_complete '
                                'num_samples_collected shared num_raw_data pi '
                                'pmids owner status')

        infolist = []
        for s_id in studylist:
            study = Study(s_id)
            status = study.status
            # Just passing the email address as the name here, since
            # name is not a required field in qiita.qiita_user
            owner = study_person_linkifier((study.owner, study.owner))
            info = study.info
            PI = StudyPerson(info['principal_investigator_id'])
            PI = study_person_linkifier((PI.email, PI.name))
            pmids = ", ".join([pubmed_linkifier([pmid])
                               for pmid in study.pmids])
            shared = _get_shared_links_for_study(study)
            infolist.append(StudyTuple(study.id, study.title,
                                       info["metadata_complete"],
                                       info["number_samples_collected"],
                                       shared, len(study.raw_data()),
                                       PI, pmids, owner, status))
        return infolist


def check_access(user, study, no_public=False):
    """make sure user has access to the study requested"""
    if not study.has_access(user, no_public):
        if no_public:
            return False
        else:
            raise HTTPError(403, "User %s does not have access to study %d" %
                                 (user.id, study.id))
    return True


def _check_owner(user, study):
    """make sure user is the owner of the study requested"""
    if not user == study.owner:
        raise HTTPError(403, "User %s does not own study %d" %
                        (user, study.id))


class CreateStudyForm(Form):
    study_title = StringField('Study Title', [validators.required()])
    study_alias = StringField('Study Alias', [validators.required()])
    pubmed_id = StringField('PubMed ID')

    # TODO:This can be filled from the database
    # in oracle, this is in controlled_vocabs (ID 1),
    #                       controlled_vocab_values with CVV IDs >= 0
    environmental_packages = SelectMultipleField(
        'Environmental Packages',
        [validators.required()],
        choices=[('air', 'air'),
                 ('host_associated', 'host-associated'),
                 ('human_amniotic_fluid', 'human-amniotic-fluid'),
                 ('human_associated', 'human-associated'),
                 ('human_blood', 'human-blood'),
                 ('human_gut', 'human-gut'),
                 ('human_oral', 'human-oral'),
                 ('human_skin', 'human-skin'),
                 ('human_urine', 'human-urine'),
                 ('human_vaginal', 'human-vaginal'),
                 ('biofilm', 'microbial mat/biofilm'),
                 ('misc_env',
                  'miscellaneous natural or artificial environment'),
                 ('plant_associated', 'plant-associated'),
                 ('sediment', 'sediment'),
                 ('soil', 'soil'),
                 ('wastewater_sludge', 'wastewater/sludge'),
                 ('water', 'water')])
    is_timeseries = BooleanField('Includes Event-Based Data')
    study_abstract = TextAreaField('Study Abstract', [validators.required()])
    study_description = StringField('Study Description',
                                    [validators.required()])
    # The choices for these "people" fields will be filled from the database
    principal_investigator = SelectField('Principal Investigator',
                                         [validators.required()],
                                         coerce=lambda x: x)
    lab_person = SelectField('Lab Person', coerce=lambda x: x)


class PrivateStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        user = User(self.current_user)
        user_studies = yield Task(self._get_private, user)
        shared_studies = yield Task(self._get_shared, user)
        all_emails_except_current = yield Task(self._get_all_emails)
        all_emails_except_current.remove(self.current_user)
        self.render('private_studies.html', user=self.current_user,
                    user_studies=user_studies, shared_studies=shared_studies,
                    all_emails_except_current=all_emails_except_current)

    def _get_private(self, user, callback):
        callback(_build_study_info("private", user))

    def _get_shared(self, user, callback):
        """builds list of tuples for studies that are shared with user"""
        callback(_build_study_info("shared", user))

    def _get_all_emails(self, callback):
        callback(list(User.iter()))


class PublicStudiesHandler(BaseHandler):
    @authenticated
    @coroutine
    def get(self):
        self.write(self.render_string('waiting.html'))
        self.flush()
        public_studies = yield Task(self._get_public)
        self.render('public_studies.html', user=self.current_user,
                    public_studies=public_studies)

    def _get_public(self, callback):
        """builds list of tuples for studies that are public"""
        callback(_build_study_info("public"))


class StudyDescriptionHandler(BaseHandler):
    def get_raw_data(self, rdis, callback):
        """Get all raw data objects from a list of raw_data_ids"""
        callback([RawData(rdi) for rdi in rdis])

    def get_prep_templates(self, raw_data, callback):
        """Get all prep templates for a list of raw data objects"""
        d = {}
        for rd in raw_data:
            # We neeed this so PrepTemplate(p) doesn't fail if that raw
            # doesn't exist but raw data has the row: #554
            prep_templates = sorted(rd.prep_templates)
            d[rd.id] = [PrepTemplate(p) for p in prep_templates
                        if PrepTemplate.exists(p)]
        callback(d)

    def remove_add_study_template(self, raw_data, study_id, fp_rsp, callback):
        """Removes prep templates, raw data and sample template and adds
           a new one
        """
        for rd in raw_data():
            if PrepTemplate.exists((rd)):
                PrepTemplate.delete(rd)
        if SampleTemplate.exists(study_id):
            SampleTemplate.delete(study_id)

        SampleTemplate.create(load_template_to_dataframe(fp_rsp),
                              Study(study_id))
        # TODO: do not remove but move to final storage space
        # and keep there forever, issue #550
        remove(fp_rsp)

        callback()

    def get_raw_data_from_other_studies(self, user, study, callback):
        """Retrieves a tuple of raw_data_id and the last study title for that
        raw_data
        """
        d = {}
        for sid in user.private_studies:
            if sid == study.id:
                continue
            for rdid in Study(sid).raw_data():
                d[rdid] = Study(RawData(rdid).studies[-1]).title
        callback(d)

    @coroutine
    def display_template(self, study_id, msg, msg_level, tab_to_display=""):
        """Simple function to avoid duplication of code"""
        # make sure study is accessible and exists, raise error if not
        study = None
        study_id = int(study_id)
        try:
            study = Study(study_id)
        except QiitaDBUnknownIDError:
            # Study not in database so fail nicely
            raise HTTPError(404, "Study %d does not exist" % study_id)
        else:
            check_access(User(self.current_user), study)

        # processing files from upload path
        fp = get_study_fp(study_id)
        if exists(fp):
            fs = listdir(fp)
        else:
            fs = []
        # getting raw filepath_ types
        fts = [k.split('_', 1)[1].replace('_', ' ')
               for k in get_filepath_types() if k.startswith('raw_')]
        fts = ['<option value="%s">%s</option>' % (f, f) for f in fts]

        study = Study(study_id)
        user = User(self.current_user)
        # getting the RawData and its prep templates
        available_raw_data = yield Task(self.get_raw_data, study.raw_data())
        available_prep_templates = yield Task(self.get_prep_templates,
                                              available_raw_data)

        # other general vars, note that we create the select options here
        # so we do not have to loop several times over them in the template
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
        data_types = ['<option value="%s">%s</option>' % (v, k)
                      for k, v in data_types]
        filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1))
        filetypes = ['<option value="%s">%s</option>' % (v, k)
                     for k, v in filetypes]
        other_studies_rd = yield Task(self.get_raw_data_from_other_studies,
                                      user, study)
        other_studies_rd = ['<option value="%s">%s</option>' % (k,
                            "id: %d, study: %s" % (k, v))
                            for k, v in viewitems(other_studies_rd)]

        ontology = Ontology(convert_to_id('ENA', 'ontology'))

        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        self.render('study_description.html', user=self.current_user,
                    study_title=study.title, study_info=study.info,
                    study_id=study_id, files=fs, filetypes=''.join(filetypes),
                    user_level=user.level, data_types=''.join(data_types),
                    available_raw_data=available_raw_data,
                    available_prep_templates=available_prep_templates,
                    ste=SampleTemplate.exists(study_id),
                    filepath_types=''.join(fts), ena_terms=''.join(ena_terms),
                    tab_to_display=tab_to_display,
                    level=msg_level, message=msg,
                    can_upload=check_access(user, study, True),
                    other_studies_rd=''.join(other_studies_rd),
                    user_defined_terms=user_defined_terms)

    @authenticated
    def get(self, study_id):
        study_id = int(study_id)
        check_access(User(self.current_user), Study(study_id))
        self.display_template(int(study_id), "", "")

    @authenticated
    @coroutine
    def post(self, study_id):
        study_id = int(study_id)
        check_access(User(self.current_user), Study(study_id))

        # vars to add sample template
        sample_template = self.get_argument('sample_template', None)
        # vars to add raw data
        filetype = self.get_argument('filetype', None)
        previous_raw_data = self.get_argument('previous_raw_data', None)
        # vars to add prep template
        add_prep_template = self.get_argument('add_prep_template', None)
        raw_data_id = self.get_argument('raw_data_id', None)
        data_type_id = self.get_argument('data_type_id', None)
        investigation_type = self.get_argument('investigation-type', None)
        user_defined_investigation_type = self.get_argument(
            'user-defined-investigation-type', None)
        new_investigation_type = self.get_argument('new-investigation-type',
                                                   None)

        # non selected is the equivalent to the user not specifyin the info
        # thus we should make the investigation_type None
        if investigation_type == "" or investigation_type == "Non selected":
            investigation_type = None

        # to update investigation type
        update_investigation_type = self.get_argument(
            'update_investigation_type', None)
        edit_investigation_type = self.get_argument('edit-investigation-type',
                                                    None)
        edit_user_defined_investigation_type = self.get_argument(
            'edit-user-defined-investigation-type', None)
        edit_new_investigation_type = self.get_argument(
            'edit-new-investigation-type', None)

        study = Study(study_id)
        msg_level = 'success'
        if sample_template:
            # processing sample templates

            fp_rsp = join(get_study_fp(study_id), sample_template)
            if not exists(fp_rsp):
                raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)

{'fraction_non_alphanumeric': 0.10446811893782795, 'fraction_numerical': 0.0021625059627921766, 'mean_word_length': 2.2140228945216682, 'pattern_counts': {'":': 8, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2347921', 'n_tokens_mistral': 8908, 'n_tokens_neox': 8486, 'n_words': 2159}