<commit_before>"""
This module is responsible for embedding images and videos that are linked
directly (as plain files)
"""

import re

image_regex = re.compile(r'^[^?]+\.(svg|png|jpe?g|gif)(?:\?.*)?$', re.IGNORECASE)
video_regex = re.compile(r'^[^?]+\.(webm|mp4)(?:\?.*)?$', re.IGNORECASE)

def embed(submission):
    # img tags don't require a type so we can take a shortcut here
    if image_regex.search(submission.url):
        return {
            'kind' : 'IMAGE',
            'url' : submission.url
        }
    # video source tags do require a type so we have to check the match to
    # find it
    match = video_regex.search(submission.url)
    if not match:
        return None

    # TODO: the regex could probably be dynamically generated from a mime
    # type -> file extension mapping
    if match.group().lower() == 'webm':
        mime = 'video/webm'
    elif match.group().lower() == 'mp4':
        mime = 'video/mp4'
    else:
        # something terrible has happened
        return None
    return {
        'kind' : 'VIDEO',
        'sources' : [{
            'mime' : mime,
            'url' : submission.url
        }]
    }<commit_msg>Fix regex api bug in plain provider<commit_after>"""
This module is responsible for embedding images and videos that are linked
directly (as plain files)
"""

import re

image_regex = re.compile(r'^[^?]+\.(svg|png|jpe?g|gif)(?:\?.*)?$', re.IGNORECASE)
video_regex = re.compile(r'^[^?]+\.(webm|mp4)(?:\?.*)?$', re.IGNORECASE)

def embed(submission):
    # img tags don't require a type so we can take a shortcut here
    if image_regex.search(submission.url):
        return {
            'kind' : 'IMAGE',
            'url' : submission.url
        }
    # video source tags do require a type so we have to check the match to
    # find it
    match = video_regex.search(submission.url)
    if not match:
        return None

    # TODO: the regex could probably be dynamically generated from a mime
    # type -> file extension mapping
    if match.group(1).lower() == 'webm':
        mime = 'video/webm'
    elif match.group(1).lower() == 'mp4':
        mime = 'video/mp4'
    else:
        # something terrible has happened
        return None
    return {
        'kind' : 'VIDEO',
        'sources' : [{
            'mime' : mime,
            'url' : submission.url
        }]
    }
{'fraction_non_alphanumeric': 0.13787234042553193, 'fraction_numerical': 0.003404255319148936, 'mean_word_length': 2.363376251788269, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3061981', 'n_tokens_mistral': 749, 'n_tokens_neox': 725, 'n_words': 228}