<commit_before>#!/usr/bin/env python
from PIL import Image
import rethinkdb as r
from optparse import OptionParser
import sys
import os
import errno
import traceback


def datafile_dir(mcdir, datafile_id):
    pieces = datafile_id.split("-")
    return os.path.join(mcdir, pieces[1][0:2], pieces[1][2:4])


def mkdirp(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise


def convert_image(f):
    if f["mediatype"]["mime"] == "image/tiff":
        return True
    elif f["mediatype"]["mime"] == "image/x-ms-bmp":
        return True
    else:
        return False


def main():
    parser = OptionParser()<commit_msg>Add image/bmp support.<commit_after>#!/usr/bin/env python
from PIL import Image
import rethinkdb as r
from optparse import OptionParser
import sys
import os
import errno
import traceback


def datafile_dir(mcdir, datafile_id):
    pieces = datafile_id.split("-")
    return os.path.join(mcdir, pieces[1][0:2], pieces[1][2:4])


def mkdirp(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise


def convert_image(f):
    if f["mediatype"]["mime"] == "image/tiff":
        return True
    elif f["mediatype"]["mime"] == "image/x-ms-bmp":
        return True
    elif f["mediatype"]["mime"] == "image/bmp":
        return True
    else:
        return False


def main():
    parser = OptionParser()
{'fraction_non_alphanumeric': 0.12874442319949012, 'fraction_numerical': 0.0076481835564053535, 'mean_word_length': 2.738095238095238, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2204869', 'n_tokens_mistral': 540, 'n_tokens_neox': 530, 'n_words': 138}