<commit_before># -*- coding: utf-8 -*-

# Tested on Markdown 2.3.1
#
# Copyright (c) 2014, <NAME>
# Copyright (c) 2014, <NAME>
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.


import re

from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor


# Grab the emojis (+800) here: https://github.com/arvida/emoji-cheat-sheet.com
# This **crazy long** list was generated by walking through the emojis.png
emojis_path = "http://localhost:8000/static/img/emojis/"
emojis_set = {
    "+1", "-1", "100", "1234", "8ball", "a", "ab", "abc", "abcd", "accept", "aerial_tramway", "airplane",
    "alarm_clock", "alien", "ambulance", "anchor", "angel", "anger", "angry", "anguished", "ant", "apple",
    "aquarius", "aries", "arrows_clockwise", "arrows_counterclockwise", "arrow_backward", "arrow_double_down",
    "arrow_double_up", "arrow_down", "arrow_down_small", "arrow_forward", "arrow_heading_down", "arrow_heading_up",
    "arrow_left", "arrow_lower_left", "arrow_lower_right", "arrow_right", "arrow_right_hook", "arrow_up",
    "arrow_upper_left", "arrow_upper_right", "arrow_up_down", "arrow_up_small", "art", "articulated_lorry",
    "astonished", "atm", "b", "baby", "baby_bottle", "baby_chick", "baby_symbol", "baggage_claim", "balloon",
    "ballot_box_with_check", "bamboo", "banana", "bangbang", "bank", "barber", "bar_chart", "baseball", "basketball",
    "bath", "bathtub", "battery", "bear", "bee", "beer", "beers", "beetle", "beginner", "bell", "bento", "bicyclist",
    "bike", "bikini", "bird", "birthday", "black_circle", "black_joker", "black_nib", "black_square",
    "black_square_button", "blossom", "blowfish", "blue_book", "blue_car", "blue_heart", "blush", "boar", "boat",
    "bomb", "book", "bookmark", "bookmark_tabs", "books", "boom", "boot", "bouquet", "bow", "bowling", "bowtie",
    "boy", "bread", "bride_with_veil", "bridge_at_night", "briefcase", "broken_heart", "bug", "bulb",
    "bullettrain_front", "bullettrain_side", "bus", "busstop", "busts_in_silhouette", "bust_in_silhouette",
    "cactus", "cake", "calendar", "calling", "camel", "camera", "cancer", "candy", "capital_abcd", "capricorn",
    "car", "card_index", "carousel_horse", "cat", "cat2", "cd", "chart", "chart_with_downwards_trend",
    "chart_with_upwards_trend", "checkered_flag", "cherries", "cherry_blossom", "chestnut", "chicken",
    "children_crossing", "chocolate_bar", "christmas_tree", "church", "cinema", "circus_tent", "city_sunrise",
    "city_sunset", "cl", "clap", "clapper", "clipboard", "clock1", "clock10", "clock1030", "clock11", "clock1130",
    "clock12", "clock1230", "clock130", "clock2", "clock230", "clock3", "clock330", "clock4", "clock430", "clock5",
    "clock530", "clock6", "clock630", "clock7", "clock730", "clock8", "clock830", "clock9", "clock930",
    "closed_book", "closed_lock_with_key", "closed_umbrella", "cloud", "clubs", "cn", "cocktail", "coffee",
    "cold_sweat", "collision", "computer", "confetti_ball", "confounded", "confused", "congratulations",
    "construction", "construction_worker", "convenience_store", "cookie", "cool", "cop", "copyright", "corn",
    "couple", "couplekiss", "couple_with_heart", "cow", "cow2", "credit_card", "crocodile", "crossed_flags",
    "crown", "cry", "crying_cat_face", "crystal_ball", "cupid", "curly_loop", "currency_exchange", "curry",
    "custard", "customs", "cyclone", "dancer", "dancers", "dango", "dart", "dash", "date", "de", "deciduous_tree",<commit_msg>Fix bug #60 on github: Emojis urls problem<commit_after># -*- coding: utf-8 -*-

# Tested on Markdown 2.3.1
#
# Copyright (c) 2014, <NAME>
# Copyright (c) 2014, <NAME>
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.


import re

from django.conf import settings

from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor


# Grab the emojis (+800) here: https://github.com/arvida/emoji-cheat-sheet.com
# This **crazy long** list was generated by walking through the emojis.png
emojis_path = "{}://{}/static/img/emojis/".format(settings.SITES["api"]["scheme"], settings.SITES["api"]["domain"])
emojis_set = {
    "+1", "-1", "100", "1234", "8ball", "a", "ab", "abc", "abcd", "accept", "aerial_tramway", "airplane",
    "alarm_clock", "alien", "ambulance", "anchor", "angel", "anger", "angry", "anguished", "ant", "apple",
    "aquarius", "aries", "arrows_clockwise", "arrows_counterclockwise", "arrow_backward", "arrow_double_down",
    "arrow_double_up", "arrow_down", "arrow_down_small", "arrow_forward", "arrow_heading_down", "arrow_heading_up",
    "arrow_left", "arrow_lower_left", "arrow_lower_right", "arrow_right", "arrow_right_hook", "arrow_up",
    "arrow_upper_left", "arrow_upper_right", "arrow_up_down", "arrow_up_small", "art", "articulated_lorry",
    "astonished", "atm", "b", "baby", "baby_bottle", "baby_chick", "baby_symbol", "baggage_claim", "balloon",
    "ballot_box_with_check", "bamboo", "banana", "bangbang", "bank", "barber", "bar_chart", "baseball", "basketball",
    "bath", "bathtub", "battery", "bear", "bee", "beer", "beers", "beetle", "beginner", "bell", "bento", "bicyclist",
    "bike", "bikini", "bird", "birthday", "black_circle", "black_joker", "black_nib", "black_square",
    "black_square_button", "blossom", "blowfish", "blue_book", "blue_car", "blue_heart", "blush", "boar", "boat",
    "bomb", "book", "bookmark", "bookmark_tabs", "books", "boom", "boot", "bouquet", "bow", "bowling", "bowtie",
    "boy", "bread", "bride_with_veil", "bridge_at_night", "briefcase", "broken_heart", "bug", "bulb",
    "bullettrain_front", "bullettrain_side", "bus", "busstop", "busts_in_silhouette", "bust_in_silhouette",
    "cactus", "cake", "calendar", "calling", "camel", "camera", "cancer", "candy", "capital_abcd", "capricorn",
    "car", "card_index", "carousel_horse", "cat", "cat2", "cd", "chart", "chart_with_downwards_trend",
    "chart_with_upwards_trend", "checkered_flag", "cherries", "cherry_blossom", "chestnut", "chicken",
    "children_crossing", "chocolate_bar", "christmas_tree", "church", "cinema", "circus_tent", "city_sunrise",
    "city_sunset", "cl", "clap", "clapper", "clipboard", "clock1", "clock10", "clock1030", "clock11", "clock1130",
    "clock12", "clock1230", "clock130", "clock2", "clock230", "clock3", "clock330", "clock4", "clock430", "clock5",
    "clock530", "clock6", "clock630", "clock7", "clock730", "clock8", "clock830", "clock9", "clock930",
    "closed_book", "closed_lock_with_key", "closed_umbrella", "cloud", "clubs", "cn", "cocktail", "coffee",
    "cold_sweat", "collision", "computer", "confetti_ball", "confounded", "confused", "congratulations",
    "construction", "construction_worker", "convenience_store", "cookie", "cool", "cop", "copyright", "corn",
    "couple", "couplekiss", "couple_with_heart", "cow", "cow2", "credit_card", "crocodile", "crossed_flags",
    "crown", "cry", "crying_cat_face", "crystal_ball", "cupid", "curly_loop", "currency_exchange", "curry",
    "custard", "customs", "cyclone", "dancer", "dancers", "dango", "dart", "dash", "date", "de", "deciduous_tree",
{'fraction_non_alphanumeric': 0.17881374480192602, 'fraction_numerical': 0.018384766907419567, 'mean_word_length': 6.370161290322581, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1671436', 'n_tokens_mistral': 3283, 'n_tokens_neox': 3097, 'n_words': 889}