<commit_before>#-------------------------------------------------------------------------------
import shutil
import os
import os.path
import config

#-------------------------------------------------------------------------------

DATADIR = config.DATADIR

FT_UNKNOWN = 0
FT_FAST = 1
FT_TGZ_FAST = 2
FT_SEQ_DB_NUC = 3
FT_SEQ_DB_AA = 4
FT_ASSEM = 5
FT_MAP_ASSEM = 6
FT_HMM = 7

fileExtTable = {
    'fastq': FT_FAST,
    'fq': FT_FAST,
    'fastq.tar.gz': FT_TGZ_FAST,
    'fq.tar.gz': FT_TGZ_FAST,
    'bam': FT_MAP_ASSEM,
    'sam': FT_MAP_ASSEM,
    'hmm': FT_HMM
}

fileOptionTable = {
    'undef' : FT_UNKNOWN,
    'fast' : FT_FAST,
    'tgz_fast' : FT_TGZ_FAST,
    'seq_db_nuc' : FT_SEQ_DB_NUC,
    'seq_db_aa' : FT_SEQ_DB_AA,
    'assem' : FT_ASSEM,
    'map_assem' : FT_MAP_ASSEM,
    'hmm_profile' : FT_HMM
}

#         typo            extension
# 0       Unknown
# 1       fastq           .fastq or .fq
# 2       compressed fq   .fastq.tar.gz or .fq.tar.gz
# 3       seq databases   .fas
# 4       assemblies      .fas
# 5       mapped assbl.   .bam or .sam
# 6       hmm profile     .hmm

#-------------------------------------------------------------------------------
def getUserFilename( username, filename ):
    # NOTE EK: NOT SURE THIS IS CORRECT TO CHANGE PATH FOR DATA LIKE THIS
    # BEFORE WAS:     return os.path.join( DATADIR, username, filename )
    return os.path.join( DATADIR, username, "data", filename )

#-------------------------------------------------------------------------------
def saveFile( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename, 'w' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def clearFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        os.remove( filename + '.part' )

#-------------------------------------------------------------------------------
def saveFilePart( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename + '.part', 'a' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def endFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        shutil.move( filename + '.part', filename )

#-------------------------------------------------------------------------------
def getFileType( filename, option ):
    if option == 'auto':
        lname = filename.lower()
        for k,v in fileExtTable.items():
            if lname.endswith( k ):
                return v
        return FT_UNKNOWN

    return fileOptionTable.get( option.lower(), FT_UNKNOWN )

#-------------------------------------------------------------------------------
<commit_msg>minor comment fix<commit_after>#-------------------------------------------------------------------------------
import shutil
import os
import os.path
import config

#-------------------------------------------------------------------------------

DATADIR = config.DATADIR

FT_UNKNOWN = 0
FT_FAST = 1
FT_TGZ_FAST = 2
FT_SEQ_DB_NUC = 3
FT_SEQ_DB_AA = 4
FT_ASSEM = 5
FT_MAP_ASSEM = 6
FT_HMM = 7

fileExtTable = {
    'fastq': FT_FAST,
    'fq': FT_FAST,
    'fastq.tar.gz': FT_TGZ_FAST,
    'fq.tar.gz': FT_TGZ_FAST,
    'bam': FT_MAP_ASSEM,
    'sam': FT_MAP_ASSEM,
    'hmm': FT_HMM
}

fileOptionTable = {
    'undef' : FT_UNKNOWN,
    'fast' : FT_FAST,
    'tgz_fast' : FT_TGZ_FAST,
    'seq_db_nuc' : FT_SEQ_DB_NUC,
    'seq_db_aa' : FT_SEQ_DB_AA,
    'assem' : FT_ASSEM,
    'map_assem' : FT_MAP_ASSEM,
    'hmm_profile' : FT_HMM
}

#         typo            extension
# 0       Unknown
# 1       fastq           .fastq or .fq
# 2       compressed fq   .fastq.tar.gz or .fq.tar.gz
# 3       seq databases nuc   .fas
# 4       seq db AA       .fas
# 5       assemblies      .fas
# 6       mapped assbl.   .bam or .sam
# 7       hmm profile     .hmm

#-------------------------------------------------------------------------------
def getUserFilename( username, filename ):
    # NOTE EK: NOT SURE THIS IS CORRECT TO CHANGE PATH FOR DATA LIKE THIS
    # BEFORE WAS:     return os.path.join( DATADIR, username, filename )
    return os.path.join( DATADIR, username, "data", filename )

#-------------------------------------------------------------------------------
def saveFile( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename, 'w' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def clearFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        os.remove( filename + '.part' )

#-------------------------------------------------------------------------------
def saveFilePart( filename, filedata ):
    dir = os.path.dirname( filename )
    if not os.path.isdir( dir ):
        os.makedirs( dir )

    with open( filename + '.part', 'a' ) as f:
        shutil.copyfileobj( filedata, f )

#-------------------------------------------------------------------------------
def endFilePart( filename ):
    if os.path.isfile( filename + '.part' ):
        shutil.move( filename + '.part', filename )

#-------------------------------------------------------------------------------
def getFileType( filename, option ):
    if option == 'auto':
        lname = filename.lower()
        for k,v in fileExtTable.items():
            if lname.endswith( k ):
                return v
        return FT_UNKNOWN

    return fileOptionTable.get( option.lower(), FT_UNKNOWN )

#-------------------------------------------------------------------------------

{'fraction_non_alphanumeric': 0.3327070879590094, 'fraction_numerical': 0.00529461998292058, 'mean_word_length': 1.1240478781284005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898297', 'n_tokens_mistral': 1774, 'n_tokens_neox': 1693, 'n_words': 432}