<commit_before># encoding: UTF-8

require 'set'

module Twitter
  # A module for including Tweet auto-linking in a class. The primary use of this is for helpers/views so they can auto-link
  # usernames, lists, hashtags and URLs.
  module Autolink extend self
    # Add <tt><a></a></tt> tags around the usernames, lists, hashtags and URLs in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash:
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::      class to add to all <tt><a></tt> tags
    # <tt>:list_class</tt>::     class to add to list <tt><a></tt> tags
    # <tt>:username_class</tt>:: class to add to username <tt><a></tt> tags
    # <tt>:hashtag_class</tt>::  class to add to hashtag <tt><a></tt> tags
    # <tt>:username_url_base</tt>::  the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:list_url_base</tt>::      the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:hashtag_url_base</tt>::   the value for <tt>href</tt> attribute on hashtag links. The <tt>#hashtag</tt> (minus the <tt>#</tt>) will be appended at the end of this.
    # <tt>:username_include_symbol</tt>:: place the <tt>@</tt> symbol within username and list links
    # <tt>:suppress_lists</tt>::          disable auto-linking to lists
    # <tt>:suppress_no_follow</tt>::      do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link(text, options = {}, &block)
      auto_link_entities(text, Extractor.extract_entities_with_indices(text, :extract_url_without_protocol => false), options, &block)
    end

    # Add <tt><a></a></tt> tags around the usernames and lists in the provided <tt>text</tt>. The
    # <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::      class to add to all <tt><a></tt> tags
    # <tt>:list_class</tt>::     class to add to list <tt><a></tt> tags
    # <tt>:username_class</tt>:: class to add to username <tt><a></tt> tags
    # <tt>:username_url_base</tt>:: the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:list_url_base</tt>::     the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:username_include_symbol</tt>:: place the <tt>@</tt> symbol within username and list links
    # <tt>:suppress_lists</tt>::          disable auto-linking to lists
    # <tt>:suppress_no_follow</tt>::      do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_usernames_or_lists(text, options = {}, &block) # :yields: list_or_username
      auto_link_entities(text, Extractor.extract_mentions_or_lists_with_indices(text), options, &block)
    end

    # Add <tt><a></a></tt> tags around the hashtags in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::     class to add to all <tt><a></tt> tags
    # <tt>:hashtag_class</tt>:: class to add to hashtag <tt><a></tt> tags
    # <tt>:hashtag_url_base</tt>:: the value for <tt>href</tt> attribute. The hashtag text (minus the <tt>#</tt>) will be appended at the end of this.
    # <tt>:suppress_no_follow</tt>:: do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_hashtags(text, options = {}, &block)  # :yields: hashtag_text
      auto_link_entities(text, Extractor.extract_hashtags_with_indices(text), options, &block)
    end

    # Add <tt><a></a></tt> tags around the URLs in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:suppress_no_follow</tt>:: do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_urls(text, options = {}, &block)
      auto_link_entities(text, Extractor.extract_urls_with_indices(text, :extract_url_without_protocol => false), options, &block)
    end

    # These methods are deprecated, will be removed in future.
    extend Deprecation

    # <b>Deprecated</b>: Please use auto_link_urls instead.
    # Add <tt><a></a></tt> tags around the URLs in the provided <tt>text</tt>.
    # Any elements in the <tt>href_options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    # Unless <tt>href_options</tt> contains <tt>:suppress_no_follow</tt>
    # the <tt>rel="nofollow"</tt> attribute will be added.
    alias :auto_link_urls_custom :auto_link_urls
    deprecate :auto_link_urls_custom, :auto_link_urls

    private

    DEFAULT_OPTIONS = {
      # Default CSS class for auto-linked URLs
      :url_class      => "tweet-url",
      # Default CSS class for auto-linked lists (along with the url class)
      :list_class     => "list-slug",
      # Default CSS class for auto-linked usernames (along with the url class)
      :username_class => "username",
      # Default CSS class for auto-linked hashtags (along with the url class)
      :hashtag_class  => "hashtag",

      # Default URL base for auto-linked usernames
      :username_url_base => "https://twitter.com/",
      # Default URL base for auto-linked lists
      :list_url_base     => "https://twitter.com/",
      # Default URL base for auto-linked hashtags
      :hashtag_url_base  => "https://twitter.com/#!/search?q=%23",
    }.freeze

    def auto_link_entities(text, entities, options = {}, &block)
      return text if entities.empty?

      # NOTE deprecate these attributes not options keys in options hash, then use html_attrs
      options = DEFAULT_OPTIONS.merge(options)
      options[:html_attrs] = extract_html_attrs_from_options!(options)
      options[:html_attrs][:rel] ||= "nofollow" unless options[:suppress_no_follow]

      Twitter::Rewriter.rewrite_entities(text, entities) do |entity, chars|
        if entity[:url]
          link_to_url(entity, chars, options, &block)
        elsif entity[:hashtag]
          link_to_hashtag(entity, chars, options, &block)
        elsif entity[:screen_name]
          link_to_screen_name(entity, chars, options, &block)
        end
      end
    end

    HTML_ENTITIES = {
      '&' => '&amp;',
      '>' => '&gt;',
      '<' => '&lt;',
      '"' => '&quot;',
      "'" => '&#39;'
    }

    def html_escape(text)
      text && text.to_s.gsub(/[&"'><]/) do |character|
        HTML_ENTITIES[character]
      end
    end

    # NOTE We will make this private in future.
    public :html_escape

    # Options which should not be passed as HTML attributes<commit_msg>Revert a method as public.<commit_after># encoding: UTF-8

require 'set'

module Twitter
  # A module for including Tweet auto-linking in a class. The primary use of this is for helpers/views so they can auto-link
  # usernames, lists, hashtags and URLs.
  module Autolink extend self
    # Default CSS class for auto-linked URLs
    DEFAULT_URL_CLASS = "tweet-url".freeze
    # Default CSS class for auto-linked lists (along with the url class)
    DEFAULT_LIST_CLASS = "list-slug".freeze
    # Default CSS class for auto-linked usernames (along with the url class)
    DEFAULT_USERNAME_CLASS = "username".freeze
    # Default CSS class for auto-linked hashtags (along with the url class)
    DEFAULT_HASHTAG_CLASS = "hashtag".freeze

    # Default URL base for auto-linked usernames
    DEFAULT_USERNAME_URL_BASE = "https://twitter.com/".freeze
    # Default URL base for auto-linked lists
    DEFAULT_LIST_URL_BASE = "https://twitter.com/".freeze
    # Default URL base for auto-linked hashtags
    DEFAULT_HASHTAG_URL_BASE = "https://twitter.com/#!/search?q=%23".freeze

    DEFAULT_OPTIONS = {
      :url_class      => DEFAULT_URL_CLASS,
      :list_class     => DEFAULT_LIST_CLASS,
      :username_class => DEFAULT_USERNAME_CLASS,
      :hashtag_class  => DEFAULT_HASHTAG_CLASS,

      :username_url_base => DEFAULT_USERNAME_URL_BASE,
      :list_url_base     => DEFAULT_LIST_URL_BASE,
      :hashtag_url_base  => DEFAULT_HASHTAG_URL_BASE
    }.freeze

    def auto_link_entities(text, entities, options = {}, &block)
      return text if entities.empty?

      # NOTE deprecate these attributes not options keys in options hash, then use html_attrs
      options = DEFAULT_OPTIONS.merge(options)
      options[:html_attrs] = extract_html_attrs_from_options!(options)
      options[:html_attrs][:rel] ||= "nofollow" unless options[:suppress_no_follow]

      Twitter::Rewriter.rewrite_entities(text, entities) do |entity, chars|
        if entity[:url]
          link_to_url(entity, chars, options, &block)
        elsif entity[:hashtag]
          link_to_hashtag(entity, chars, options, &block)
        elsif entity[:screen_name]
          link_to_screen_name(entity, chars, options, &block)
        end
      end
    end

    # Add <tt><a></a></tt> tags around the usernames, lists, hashtags and URLs in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash:
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::      class to add to all <tt><a></tt> tags
    # <tt>:list_class</tt>::     class to add to list <tt><a></tt> tags
    # <tt>:username_class</tt>:: class to add to username <tt><a></tt> tags
    # <tt>:hashtag_class</tt>::  class to add to hashtag <tt><a></tt> tags
    # <tt>:username_url_base</tt>::  the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:list_url_base</tt>::      the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:hashtag_url_base</tt>::   the value for <tt>href</tt> attribute on hashtag links. The <tt>#hashtag</tt> (minus the <tt>#</tt>) will be appended at the end of this.
    # <tt>:username_include_symbol</tt>:: place the <tt>@</tt> symbol within username and list links
    # <tt>:suppress_lists</tt>::          disable auto-linking to lists
    # <tt>:suppress_no_follow</tt>::      do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link(text, options = {}, &block)
      auto_link_entities(text, Extractor.extract_entities_with_indices(text, :extract_url_without_protocol => false), options, &block)
    end

    # Add <tt><a></a></tt> tags around the usernames and lists in the provided <tt>text</tt>. The
    # <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::      class to add to all <tt><a></tt> tags
    # <tt>:list_class</tt>::     class to add to list <tt><a></tt> tags
    # <tt>:username_class</tt>:: class to add to username <tt><a></tt> tags
    # <tt>:username_url_base</tt>:: the value for <tt>href</tt> attribute on username links. The <tt>@username</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:list_url_base</tt>::     the value for <tt>href</tt> attribute on list links. The <tt>@username/list</tt> (minus the <tt>@</tt>) will be appended at the end of this.
    # <tt>:username_include_symbol</tt>:: place the <tt>@</tt> symbol within username and list links
    # <tt>:suppress_lists</tt>::          disable auto-linking to lists
    # <tt>:suppress_no_follow</tt>::      do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_usernames_or_lists(text, options = {}, &block) # :yields: list_or_username
      auto_link_entities(text, Extractor.extract_mentions_or_lists_with_indices(text), options, &block)
    end

    # Add <tt><a></a></tt> tags around the hashtags in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:url_class</tt>::     class to add to all <tt><a></tt> tags
    # <tt>:hashtag_class</tt>:: class to add to hashtag <tt><a></tt> tags
    # <tt>:hashtag_url_base</tt>:: the value for <tt>href</tt> attribute. The hashtag text (minus the <tt>#</tt>) will be appended at the end of this.
    # <tt>:suppress_no_follow</tt>:: do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_hashtags(text, options = {}, &block)  # :yields: hashtag_text
      auto_link_entities(text, Extractor.extract_hashtags_with_indices(text), options, &block)
    end

    # Add <tt><a></a></tt> tags around the URLs in the provided <tt>text</tt>.
    # The <tt><a></tt> tags can be controlled with the following entries in the <tt>options</tt> hash.
    # Also any elements in the <tt>options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    #
    # <tt>:suppress_no_follow</tt>:: do not add <tt>rel="nofollow"</tt> to auto-linked items
    def auto_link_urls(text, options = {}, &block)
      auto_link_entities(text, Extractor.extract_urls_with_indices(text, :extract_url_without_protocol => false), options, &block)
    end

    # These methods are deprecated, will be removed in future.
    extend Deprecation

    # <b>Deprecated</b>: Please use auto_link_urls instead.
    # Add <tt><a></a></tt> tags around the URLs in the provided <tt>text</tt>.
    # Any elements in the <tt>href_options</tt> hash will be converted to HTML attributes
    # and place in the <tt><a></tt> tag.
    # Unless <tt>href_options</tt> contains <tt>:suppress_no_follow</tt>
    # the <tt>rel="nofollow"</tt> attribute will be added.
    alias :auto_link_urls_custom :auto_link_urls
    deprecate :auto_link_urls_custom, :auto_link_urls

    private

    HTML_ENTITIES = {
      '&' => '&amp;',
      '>' => '&gt;',
      '<' => '&lt;',
      '"' => '&quot;',
      "'" => '&#39;'
    }

    def html_escape(text)
      text && text.to_s.gsub(/[&"'><]/) do |character|
        HTML_ENTITIES[character]
      end
    end

    # NOTE We will make this private in future.
    public :html_escape

    # Options which should not be passed as HTML attributes
{'fraction_non_alphanumeric': 0.13887601774012048, 'fraction_numerical': 0.0006619447938041968, 'mean_word_length': 3.4487632508833923, 'pattern_counts': {'":': 0, '<': 411, '<?xml version=': 0, '>': 439, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3462441', 'n_tokens_mistral': 4928, 'n_tokens_neox': 4853, 'n_words': 1511}