<commit_before># encoding: utf-8

# Original source is "青空文庫→TeX（ver. 0.9.5 2004/5/5 psitau）"
# see: http://psitau.kitunebi.com/aozora.html
#
# Also another source is "青空キンドル [Beta]"
# see: http://a2k.aill.org/

require 'nkf'

class Aozora4Reader

  PreambleLineNumber=13
  KANJIPAT = "[々〇〻\u3400-\u9FFF\uF900-\uFAFF※ヶ]"

  def self.a4r(file)
    self.new(file).main
  end

  def initialize(file)
    @inputfile_name = file

    @jisage = false
    @log_text = []
    @line_num=0
    @gaiji = {}
  end

  # UTF-8で出力
  def normalize(l)
    ##l.gsub!(/&/, '\\\\&')
    l.to_s
  end
  
  # 全角→半角
  def to_single_byte(str)
    s = str.dup
    if s =~ /[０-９]/
      s.tr!("１２３４５６７８９０", "1234567890")
    elsif s =~ /[一二三四五六七八九〇]/
      s.tr!("一二三四五六七八九〇", "1234567890")
    end
    case s
    when /\d十\d/
      s.sub!(/(\d)十(\d)/, '\1\2')
    when /\d十/
      s.sub!(/(\d)十/, '\{1}0')
    when /十\d/
      s.sub!(/十(\d)/, '1\1')
    when /十/
      s.sub!(/十/, "10")
    end
    if s =~/[！？]/
      s.tr!("！？", "!?")
    end

    return s
  end
  

  # プリアンブルの出力
  def preamble
    str = <<"END_OF_PRE"
\\documentclass[a5paper]{tbook}
%\\documentclass[a5paper, twocolumn]{tbook}
%\\usepackage[deluxe]{otf}
\\usepackage[expert, deluxe]{otf}
%\\usepackage{utf}
\\usepackage{furikana}
\\usepackage{type1cm}
\\usepackage[size=large]{aozora4reader}
\\def\\rubykatuji{\\rubyfamily\\tiny}
%\\def\\rubykatuji{\\tiny}%for UTF package
%\\renewenvironment{teihon}{\\comment}{\\endcomment}
\\usepackage[dvipdfm,bookmarks=true,bookmarksnumbered=false,%
            pdftitle={#{@title}},%
            pdfauthor={#{@author}}]{hyperref}
%% Bookmarkの文字化け対策（日本語向け）
\\ifnum 46273=\\euc"B4C1 % 46273 == 0xB4C1 == 漢(EUC-JP)
  \\AtBeginDvi{\\special{pdf:tounicode EUC-UCS2}}%<commit_msg>remove hyperlink of footnote<commit_after># encoding: utf-8

# Original source is "青空文庫→TeX（ver. 0.9.5 2004/5/5 psitau）"
# see: http://psitau.kitunebi.com/aozora.html
#
# Also another source is "青空キンドル [Beta]"
# see: http://a2k.aill.org/

require 'nkf'

class Aozora4Reader

  PreambleLineNumber=13
  KANJIPAT = "[々〇〻\u3400-\u9FFF\uF900-\uFAFF※ヶ]"

  def self.a4r(file)
    self.new(file).main
  end

  def initialize(file)
    @inputfile_name = file

    @jisage = false
    @log_text = []
    @line_num=0
    @gaiji = {}
  end

  # UTF-8で出力
  def normalize(l)
    ##l.gsub!(/&/, '\\\\&')
    l.to_s
  end
  
  # 全角→半角
  def to_single_byte(str)
    s = str.dup
    if s =~ /[０-９]/
      s.tr!("１２３４５６７８９０", "1234567890")
    elsif s =~ /[一二三四五六七八九〇]/
      s.tr!("一二三四五六七八九〇", "1234567890")
    end
    case s
    when /\d十\d/
      s.sub!(/(\d)十(\d)/, '\1\2')
    when /\d十/
      s.sub!(/(\d)十/, '\{1}0')
    when /十\d/
      s.sub!(/十(\d)/, '1\1')
    when /十/
      s.sub!(/十/, "10")
    end
    if s =~/[！？]/
      s.tr!("！？", "!?")
    end

    return s
  end
  

  # プリアンブルの出力
  def preamble
    str = <<"END_OF_PRE"
\\documentclass[a5paper]{tbook}
%\\documentclass[a5paper, twocolumn]{tbook}
%\\usepackage[deluxe]{otf}
\\usepackage[expert, deluxe]{otf}
%\\usepackage{utf}
\\usepackage{furikana}
\\usepackage{type1cm}
\\usepackage[size=large]{aozora4reader}
\\def\\rubykatuji{\\rubyfamily\\tiny}
%\\def\\rubykatuji{\\tiny}%for UTF package
%\\renewenvironment{teihon}{\\comment}{\\endcomment}
\\usepackage[dvipdfm,bookmarks=false,bookmarksnumbered=false,hyperfootnotes=false,%
            pdftitle={#{@title}},%
            pdfauthor={#{@author}}]{hyperref}
%% Bookmarkの文字化け対策（日本語向け）
\\ifnum 46273=\\euc"B4C1 % 46273 == 0xB4C1 == 漢(EUC-JP)
  \\AtBeginDvi{\\special{pdf:tounicode EUC-UCS2}}%
{'fraction_non_alphanumeric': 0.20747821197638458, 'fraction_numerical': 0.04779308405960079, 'mean_word_length': 2.7610993657505287, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2930552', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1700, 'n_words': 242}