<commit_before>import argparse
import time
import shelve
import imp
import json
import psutil
import os
import sys
import hashlib
import datetime
import csv
import StringIO
import re
import logging
import requests

try:
    from hachoir_core import config
    from hachoir_parser import createParser
    from hachoir_metadata import extractMetadata
    HAVE_HACHOIR = True
except:
    HAVE_HACHOIR = False

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

re_notascii = re.compile('\W')

DEFAULT_CAMERA_NAME = 'unnamed' # used if we can't parse a camera name from video file names
REQUIRED_MODULE_CALLBACK_FUNCTIONS = ['register_camera', 'post_video_content']

def get_duration(filename):
    duration = None
    if HAVE_HACHOIR:
        filename = unicode(filename, "utf-8")
        parser = createParser(filename)
        metadata = extractMetadata(parser, quality=1.0)
        duration = metadata.getValues('duration')[0].total_seconds()
    return duration

class GenericImporter(object):
    
    def get_params(self, path):
        camera = epoch = lat = lng = None
        if self.regex:
            match = self.regex.match(path)
            if match:
                try:
                    camera = match.group('camera')<commit_msg>Making meta-data extraction from movies safe<commit_after>import argparse
import time
import shelve
import imp
import json
import psutil
import os
import sys
import hashlib
import datetime
import csv
import StringIO
import re
import traceback
import logging
import requests

try:
    from hachoir_core import config
    from hachoir_parser import createParser
    from hachoir_metadata import extractMetadata
    HAVE_HACHOIR = True
except:
    HAVE_HACHOIR = False

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

re_notascii = re.compile('\W')

DEFAULT_CAMERA_NAME = 'unnamed' # used if we can't parse a camera name from video file names
REQUIRED_MODULE_CALLBACK_FUNCTIONS = ['register_camera', 'post_video_content']

def get_duration(filename):
    duration = None
    try:
        if HAVE_HACHOIR:
            filename = unicode(filename, "utf-8")
            parser = createParser(filename)
            metadata = extractMetadata(parser, quality=1.0)
            duration = metadata.getValues('duration')[0].total_seconds()
        return duration
    except:
        Log.error("error while getting duration meta-data from movie (%s)", filename)
        Log.error(traceback.format_exc())
        return None

class GenericImporter(object):
    
    def get_params(self, path):
        camera = epoch = lat = lng = None
        if self.regex:
            match = self.regex.match(path)
            if match:
                try:
                    camera = match.group('camera')
{'fraction_non_alphanumeric': 0.07457257184430702, 'fraction_numerical': 0.0029101491451436886, 'mean_word_length': 3.147812971342383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1814256', 'n_tokens_mistral': 781, 'n_tokens_neox': 770, 'n_words': 232}