<commit_before>class MarkupTweet
  class << self
    def markup_tweet(tweet)
      text = tweet['text']
      entities = tweet['entities']
      MarkupTweet::markup_media(text, entities)
      MarkupTweet::markup_urls(text, entities)
      MarkupTweet::markup_user_mentions(text, entities)
      MarkupTweet::markup_hashtags(text, entities)
      text
    end

    # see https://dev.twitter.com/docs/tweet-entities
    def markup_media(text, entities)
      return text unless entities['media']
      entities['media'].each do |image|
        text << "<div><a href='#{image['display_url']}'><img src='#{image['media_url']}' /></a></div>"
      end
      text
    end

    def markup_urls(text, entities)
      entities['urls'].each do |url|
        new_url = url['expanded_url'] || url['url']
        text.gsub!(url['url'], "<a href='#{new_url}'>#{new_url}</a>")
      end
      text
    end

    def markup_user_mentions(text, entities)
      entities['user_mentions'].each do |mention|
        screen_name = mention['screen_name']
        text.gsub!("@#{screen_name}", "<a href='http://twitter.com/#{screen_name}'>@#{screen_name}</a>")
      end
      text
    end

    def markup_hashtags(text, entities)
      entities['hashtags'].each do |hashtag|
        hashtag_text = hashtag['text']
        text.gsub!(/[\#＃♯]#{Regexp.quote hashtag_text}/, "<a href='http://twitter.com/search?q=%23#{hashtag_text}'>##{hashtag_text}</a>")<commit_msg>Support multiple photos<commit_after>class MarkupTweet
  class << self
    def markup_tweet(tweet)
      text = tweet['text']
      entities = tweet['entities']
      extended_entities = tweet['extended_entities']
      MarkupTweet::markup_media(text, entities, extended_entities)
      MarkupTweet::markup_urls(text, entities)
      MarkupTweet::markup_user_mentions(text, entities)
      MarkupTweet::markup_hashtags(text, entities)
      text
    end

    # see https://dev.twitter.com/docs/tweet-entities
    def markup_media(text, entities, extended_entities)
      en = extended_entities && extended_entities['media'] ? extended_entities : entities
      return text unless en['media']
      en['media'].each do |media|
        next unless media['type'] == 'photo'
        text << "<div><a href='#{media['display_url']}'><img src='#{media['media_url']}' /></a></div>"
      end
      text
    end

    def markup_urls(text, entities)
      entities['urls'].each do |url|
        new_url = url['expanded_url'] || url['url']
        text.gsub!(url['url'], "<a href='#{new_url}'>#{new_url}</a>")
      end
      text
    end

    def markup_user_mentions(text, entities)
      entities['user_mentions'].each do |mention|
        screen_name = mention['screen_name']
        text.gsub!("@#{screen_name}", "<a href='http://twitter.com/#{screen_name}'>@#{screen_name}</a>")
      end
      text
    end

    def markup_hashtags(text, entities)
      entities['hashtags'].each do |hashtag|
        hashtag_text = hashtag['text']
        text.gsub!(/[\#＃♯]#{Regexp.quote hashtag_text}/, "<a href='http://twitter.com/search?q=%23#{hashtag_text}'>##{hashtag_text}</a>")
{'fraction_non_alphanumeric': 0.16741790083708952, 'fraction_numerical': 0.00128783000643915, 'mean_word_length': 3.413352272727273, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '724713', 'n_tokens_mistral': 1039, 'n_tokens_neox': 1056, 'n_words': 176}