<commit_before>##
# Copyright 2017 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#	See the License for the specific language governing permissions and
#	limitations under the License.
##
require_relative('links/audio')
require_relative('links/image')
require_relative('links/video')

module WikiThat
  ##
  # Lexer module for links and embedded media
  #
  # This module deviates from standard MediaWiki in its use
  # of namespaces and support for media tags. It is not compliant
  # with the Wikimedia Foundation grammar.
  #
  # @author <NAME>
  ##
  module Links

    # Property attributes for images
    IMAGE_ATTRIBUTES = %w(border frame left right center thumb thumbnail)

    ##
    # Parse the current text as a link to content in a namespace
    ##
    def parse_link_internal
      advance
      url = ''
      namespaces = []
      while match? [:link_namespace]
        temp = current.value
        if temp == 'http' or temp == 'https'
          break
        end
        namespaces.push(temp)
        advance
      end
      attributes = []
      while match? [:link_namespace,:text]
        url += current.value
        if match? [:link_namespace]
          url += ':'
        end
        advance
      end
      while match? [:link_attribute]
        advance
        if match? [:text]
          attributes.push(current.value)
          advance
        else
          break
        end
      end<commit_msg>Cleanup of redundant codepaths<commit_after>##
# Copyright 2017 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#	See the License for the specific language governing permissions and
#	limitations under the License.
##
require_relative('links/audio')
require_relative('links/image')
require_relative('links/video')

module WikiThat
  ##
  # Lexer module for links and embedded media
  #
  # This module deviates from standard MediaWiki in its use
  # of namespaces and support for media tags. It is not compliant
  # with the Wikimedia Foundation grammar.
  #
  # @author <NAME>
  ##
  module Links

    # Property attributes for images
    IMAGE_ATTRIBUTES = %w(border frame left right center thumb thumbnail)

    ##
    # Parse the current text as a link to content in a namespace
    ##
    def parse_link_internal
      advance
      url = ''
      namespaces = []
      while match? [:link_namespace]
        temp = current.value
        if temp == 'http' or temp == 'https'
          error 'External link in internal link brackets?'
          return
        end
        namespaces.push(temp)
        advance
      end
      attributes = []
      while match? [:text]
        url += current.value
        advance
      end
      while match? [:link_attribute]
        advance
        if match? [:text]
          attributes.push(current.value)
          advance
        end
      end
{'fraction_non_alphanumeric': 0.07707235930135847, 'fraction_numerical': 0.004435819240365955, 'mean_word_length': 2.608, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4923914', 'n_tokens_mistral': 986, 'n_tokens_neox': 964, 'n_words': 390}