<commit_before>#!ruby 
#
# Erex Yomi e-mails a featured article to a mailing list
# 
# Author: <NAME>
# Source: https://github.com/abartov/erex_yomi
# 
# This code is in the public domain.  Where not allowed, it is released under the Creative Commons Zero License (CC0).

require "rubygems"
require 'date'
require 'uri'
require 'media_wiki'
require 'action_mailer'

HTML_PROLOGUE = "<html lang=\"he\" dir=\"rtl\"><head><meta charset=\"UTF-8\" /><body>"
HTML_EPILOGUE = "</body></html>"
HEBMONTHS = [nil, 'בינואר', 'בפברואר', 'במארס', 'באפריל', 'במאי', 'ביוני', 'ביולי', 'באוגוסט', 'בספטמבר', 'באוקטובר', 'בנובמבר', 'בדצמבר']

REXML::Document.entity_expansion_text_limit = 200000

class Mailer < ActionMailer::Base
  def daily_email(body)
    mail( :to => "<EMAIL>", :from => "<EMAIL>", :subject => "תוכן מומלץ יומי מויקיפדיה - "+heb_date) do |format|
      format.html { render text: body }
    end
  end
end

def fixlinks(str)
  return str.gsub('href="/','href="http://he.wikipedia.org/').gsub('//upload.wiki','http://upload.wiki')
end

def prepare_article_part(mw)
  h = mw.render('תבנית:הערך המומלץ')
  m = /לערך המלא/.match h
  s = m.pre_match[m.pre_match.rindex('href="/wiki/')+12..-1]
  article_title = URI.unescape(s[0..s.index('"')-1])
  print "Title: #{article_title}"
  h = mw.render(article_title)
  # grab everything before the TOC
  File.open('DEBUG.txt', 'w') {|f| f.write(h) }
  m = /(<p>.*<\/p>).*<table id=\"toc\" class=\"toc\">/m.match(h)
  if m.nil?<commit_msg>removed debugging write<commit_after>#!ruby 
#
# Erex Yomi e-mails a featured article to a mailing list
# 
# Author: <NAME>
# Source: https://github.com/abartov/erex_yomi
# 
# This code is in the public domain.  Where not allowed, it is released under the Creative Commons Zero License (CC0).

require "rubygems"
require 'date'
require 'uri'
require 'media_wiki'
require 'action_mailer'

HTML_PROLOGUE = "<html lang=\"he\" dir=\"rtl\"><head><meta charset=\"UTF-8\" /><body>"
HTML_EPILOGUE = "</body></html>"
HEBMONTHS = [nil, 'בינואר', 'בפברואר', 'במארס', 'באפריל', 'במאי', 'ביוני', 'ביולי', 'באוגוסט', 'בספטמבר', 'באוקטובר', 'בנובמבר', 'בדצמבר']

REXML::Document.entity_expansion_text_limit = 200000

class Mailer < ActionMailer::Base
  def daily_email(body)
    mail( :to => "<EMAIL>", :from => "<EMAIL>", :subject => "תוכן מומלץ יומי מויקיפדיה - "+heb_date) do |format|
      format.html { render text: body }
    end
  end
end

def fixlinks(str)
  return str.gsub('href="/','href="http://he.wikipedia.org/').gsub('//upload.wiki','http://upload.wiki')
end

def prepare_article_part(mw)
  h = mw.render('תבנית:הערך המומלץ')
  m = /לערך המלא/.match h
  s = m.pre_match[m.pre_match.rindex('href="/wiki/')+12..-1]
  article_title = URI.unescape(s[0..s.index('"')-1])
  print "Title: #{article_title}"
  h = mw.render(article_title)
  # grab everything before the TOC
  m = /(<p>.*<\/p>).*<table id=\"toc\" class=\"toc\">/m.match(h)
  if m.nil?
{'fraction_non_alphanumeric': 0.19644079397672826, 'fraction_numerical': 0.00889801505817933, 'mean_word_length': 4.383057090239411, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48054', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1271, 'n_words': 261}