<commit_before>module AbstractXmlController
#TODO rename this


  ##############################################
  # All code to manipulate source transcription
  # belongs here.
  ##############################################

  #constant - words to ignore when autolinking
  STOPWORDS = ["Mrs", "Mrs.", "Mr.", "Mr", "Dr.", "Dr", "Miss", "he", "she", "it"]

  def autolink(text)
    #find the list of articles
    if @collection.is_a?(DocumentSet)
      id = @collection.collection.id
    else
      id = @collection.id
    end
    sql = 'select distinct article_id, '+
          'display_text '+
          'from page_article_links ' +
          'inner join articles a '+
          'on a.id = article_id ' +
          "where a.collection_id = #{id}"
    logger.debug(sql)
    matches =
      Page.connection.select_all(sql).to_a
    # Bug 18 -- longest possible match is best
    matches.sort! { |x,y| x['display_text'].length <=> y['display_text'].length }
    matches.reverse!
    #for each article, check text to see if it needs to be linked
    for match in matches
      match_regex = Regexp.new('\b'+Regexp.escape(match['display_text'])+'\b', Regexp::IGNORECASE)
      #match_regex = Regexp.new('\b'+match['display_text'].gsub('?', '\?').gsub('.', '\.').gsub('(', '\(').gsub(')', '\)')+'\b', Regexp::IGNORECASE)
      display_text = match['display_text']
      logger.debug("DEBUG looking for #{match_regex}")

      #if the match is a stopword, ignore it and move to the next match
      if display_text.in?(STOPWORDS)

      else
        #find the matches and substitute in as long as the text isn't already in a link
        text.gsub! match_regex do |m|
          #find the index of the match to check if it's with a larger link
          position = Regexp.last_match.offset(0)[0]
          #check to see if the regex is already within a link, from each index
          if word_not_okay(text, position, m) || within_link(text, position)
            m
  
          else
            # not within a link, so create a new one
            article = Article.find(match['article_id'].to_i)
<commit_msg>better handling of special chars in regexps<commit_after>module AbstractXmlController
#TODO rename this


  ##############################################
  # All code to manipulate source transcription
  # belongs here.
  ##############################################

  #constant - words to ignore when autolinking
  STOPWORDS = ["Mrs", "Mrs.", "Mr.", "Mr", "Dr.", "Dr", "Miss", "he", "she", "it"]

  def autolink(text)
    #find the list of articles
    if @collection.is_a?(DocumentSet)
      id = @collection.collection.id
    else
      id = @collection.id
    end
    sql = 'select distinct article_id, '+
          'display_text '+
          'from page_article_links ' +
          'inner join articles a '+
          'on a.id = article_id ' +
          "where a.collection_id = #{id}"
    logger.debug(sql)
    matches =
      Page.connection.select_all(sql).to_a
    # Bug 18 -- longest possible match is best
    matches.sort! { |x,y| x['display_text'].length <=> y['display_text'].length }
    matches.reverse!
    #for each article, check text to see if it needs to be linked
    for match in matches
      match_regex = Regexp.new('\b'+Regexp.escape(match['display_text'])+'\b', Regexp::IGNORECASE)
      display_text = match['display_text']
      logger.debug("DEBUG looking for #{match_regex}")

      #if the match is a stopword, ignore it and move to the next match
      if display_text.in?(STOPWORDS)

      else
        #find the matches and substitute in as long as the text isn't already in a link
        text.gsub! match_regex do |m|
          #find the index of the match to check if it's with a larger link
          position = Regexp.last_match.offset(0)[0]
          #check to see if the regex is already within a link, from each index
          if word_not_okay(text, position, m) || within_link(text, position)
            m
  
          else
            # not within a link, so create a new one
            article = Article.find(match['article_id'].to_i)

{'fraction_non_alphanumeric': 0.15775335775335775, 'fraction_numerical': 0.0019536019536019536, 'mean_word_length': 2.757798165137615, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5989474', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1196, 'n_words': 391}