<commit_before>###########################################################
#
# Copyright (c) 2005, Southpaw Technology
#                     All Rights Reserved
#
# PROPRIETARY INFORMATION.  This software is proprietary to
# Southpaw Technology, and is not to be reproduced, transmitted,
# or disclosed in any way without written permission.
#
#
#

__all__ = ["FileException", "File", "FileAccess", "IconCreator", "FileGroup", "FileRange"]

from pyasm.common import Common, Xml, TacticException, Environment, System, Config, jsonloads
from pyasm.search import *
from project import Project
from subprocess import Popen, PIPE

import sys, os, string, re, stat, glob


try:
    #import Image
    from PIL import Image
    # Test to see if imaging actually works
    import _imaging
    HAS_PIL = True
except:
    HAS_PIL = False
    try:
        import Image
        # Test to see if imaging actually works
        import _imaging
        HAS_PIL = True
    except:
        HAS_PIL = False


# check if imagemagick is installed, and find exe if possible
convert_exe = ''
HAS_IMAGE_MAGICK = False
if os.name == "nt":
    # prefer direct exe to not confuse with other convert.exe present on nt systems
    convert_exe_list = glob.glob('C:\\Program Files\\ImageMagick*')
    for exe in convert_exe_list:
        try:
            convert_process = Popen(['%s\\convert.exe'%exe,'-version'], stdout=PIPE, stderr=PIPE)
            convert_return,convert_err = convert_process.communicate()
            if 'ImageMagick' in convert_return:
                convert_exe = '%s\\convert.exe'%exe
                HAS_IMAGE_MAGICK = True
        except:
            continue
        else:

            print("ImageMagick found in %s" %exe)
    if not convert_exe_list:
        # IM might not be in Program Files but may still be in PATH
        try:
            convert_process = Popen(['convert.exe','-version'], stdout=PIPE, stderr=PIPE)
            convert_return,convert_err = convert_process.communicate()
            if 'ImageMagick' in convert_return:
                convert_exe = 'convert.exe'
                HAS_IMAGE_MAGICK = True
        except:
            pass
else:
    # in other systems (e.g. unix) 'convert' is expected to be in PATH
    try:
        convert_process = Popen(['convert','-version'], stdout=PIPE, stderr=PIPE)
        convert_return,convert_err = convert_process.communicate()
        if 'ImageMagick' in convert_return:
            convert_exe = 'convert'
            HAS_IMAGE_MAGICK = True
    except:
        pass

ffprobe_exe = "ffprobe"
ffmpeg_exe = "ffmpeg"
if os.name == 'nt':
    ffprobe_exe = "ffprobe.exe"
    ffmpeg_exe = "ffmpeg.exe"

if Common.which(ffprobe_exe):
    HAS_FFMPEG = True
else:
    HAS_FFMPEG = False




import subprocess

class FileException(TacticException):
    pass


class File(SObject):

    NORMAL_EXT = ['max','ma','xls' ,'xlsx', 'doc', 'docx','txt', 'rtf', 'odt','fla','psd', 'xsi', 'scn', 'hip', 'xml','eani','pdf', 'fbx',
            'gz', 'zip', 'rar',
            'ini', 'db', 'py', 'pyd', 'spt', 'rpm', 'gz'
    ]

    VIDEO_EXT = ['mov','wmv','mpg','mpeg','m1v','m2v','mp2','mp4','mpa','mpe','mp4','wma','asf','asx','avi','wax',
                'wm','wvx','ogg','webm','mkv','m4v','mxf','f4v','rmvb', 'gif']

    #IMAGE_EXT = ['jpg','png','tif','tiff','gif','dds','dcm']
    IMAGE_EXT = ['jpg','jpeg', 'png','tif','tiff','dds','dcm']




    SEARCH_TYPE = "sthpw/file"
    BASE_TYPE_SEQ = "sequence"
    BASE_TYPE_DIR = "directory"
    BASE_TYPE_FILE = "file"



    def get_code(self):
        return self.get_value("code")

    def get_file_name(self):
        return self.get_value("file_name")

    def get_file_range(self):
        return self.get_value("file_range")

    def get_type(self):
        return self.get_value("type")


    def get_media_type_by_path(cls, path):
        tmp, ext = os.path.splitext(path)
        ext = ext.lstrip(".")
        ext = ext.lower()<commit_msg>Added .bmp to list of valid image extensions<commit_after>###########################################################
#
# Copyright (c) 2005, Southpaw Technology
#                     All Rights Reserved
#
# PROPRIETARY INFORMATION.  This software is proprietary to
# Southpaw Technology, and is not to be reproduced, transmitted,
# or disclosed in any way without written permission.
#
#
#

__all__ = ["FileException", "File", "FileAccess", "IconCreator", "FileGroup", "FileRange"]

from pyasm.common import Common, Xml, TacticException, Environment, System, Config, jsonloads
from pyasm.search import *
from project import Project
from subprocess import Popen, PIPE

import sys, os, string, re, stat, glob


try:
    #import Image
    from PIL import Image
    # Test to see if imaging actually works
    import _imaging
    HAS_PIL = True
except:
    HAS_PIL = False
    try:
        import Image
        # Test to see if imaging actually works
        import _imaging
        HAS_PIL = True
    except:
        HAS_PIL = False


# check if imagemagick is installed, and find exe if possible
convert_exe = ''
HAS_IMAGE_MAGICK = False
if os.name == "nt":
    # prefer direct exe to not confuse with other convert.exe present on nt systems
    convert_exe_list = glob.glob('C:\\Program Files\\ImageMagick*')
    for exe in convert_exe_list:
        try:
            convert_process = Popen(['%s\\convert.exe'%exe,'-version'], stdout=PIPE, stderr=PIPE)
            convert_return,convert_err = convert_process.communicate()
            if 'ImageMagick' in convert_return:
                convert_exe = '%s\\convert.exe'%exe
                HAS_IMAGE_MAGICK = True
        except:
            continue
        else:

            print("ImageMagick found in %s" %exe)
    if not convert_exe_list:
        # IM might not be in Program Files but may still be in PATH
        try:
            convert_process = Popen(['convert.exe','-version'], stdout=PIPE, stderr=PIPE)
            convert_return,convert_err = convert_process.communicate()
            if 'ImageMagick' in convert_return:
                convert_exe = 'convert.exe'
                HAS_IMAGE_MAGICK = True
        except:
            pass
else:
    # in other systems (e.g. unix) 'convert' is expected to be in PATH
    try:
        convert_process = Popen(['convert','-version'], stdout=PIPE, stderr=PIPE)
        convert_return,convert_err = convert_process.communicate()
        if 'ImageMagick' in convert_return:
            convert_exe = 'convert'
            HAS_IMAGE_MAGICK = True
    except:
        pass

ffprobe_exe = "ffprobe"
ffmpeg_exe = "ffmpeg"
if os.name == 'nt':
    ffprobe_exe = "ffprobe.exe"
    ffmpeg_exe = "ffmpeg.exe"

if Common.which(ffprobe_exe):
    HAS_FFMPEG = True
else:
    HAS_FFMPEG = False




import subprocess

class FileException(TacticException):
    pass


class File(SObject):

    NORMAL_EXT = ['max','ma','xls' ,'xlsx', 'doc', 'docx','txt', 'rtf', 'odt','fla','psd', 'xsi', 'scn', 'hip', 'xml','eani','pdf', 'fbx',
            'gz', 'zip', 'rar',
            'ini', 'db', 'py', 'pyd', 'spt', 'rpm', 'gz'
    ]

    VIDEO_EXT = ['mov','wmv','mpg','mpeg','m1v','m2v','mp2','mp4','mpa','mpe','mp4','wma','asf','asx','avi','wax',
                'wm','wvx','ogg','webm','mkv','m4v','mxf','f4v','rmvb', 'gif']

    #IMAGE_EXT = ['jpg','png','tif','tiff','gif','dds','dcm']
    IMAGE_EXT = ['jpg','jpeg', 'png','tif','tiff','dds','dcm', 'bmp']




    SEARCH_TYPE = "sthpw/file"
    BASE_TYPE_SEQ = "sequence"
    BASE_TYPE_DIR = "directory"
    BASE_TYPE_FILE = "file"



    def get_code(self):
        return self.get_value("code")

    def get_file_name(self):
        return self.get_value("file_name")

    def get_file_range(self):
        return self.get_value("file_range")

    def get_type(self):
        return self.get_value("type")


    def get_media_type_by_path(cls, path):
        tmp, ext = os.path.splitext(path)
        ext = ext.lstrip(".")
        ext = ext.lower()
{'fraction_non_alphanumeric': 0.14685138539042822, 'fraction_numerical': 0.0027707808564231737, 'mean_word_length': 2.8812316715542523, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485920', 'n_tokens_mistral': 2712, 'n_tokens_neox': 2533, 'n_words': 645}