<commit_before># Copyright (c) 2012-2020 <NAME>, <NAME>
#
# This program is free software.
# You can distribute or modify this program under the terms of
# the GNU LGPL, Lesser General Public License version 2.1.
# For details of the GNU LGPL, see the file "COPYING".
#
require 'pathname'
require 'fileutils'
require 'yaml'
require 'shellwords'

module ReVIEW
  module MakerHelper
    # Return review/bin directory
    def bindir
      Pathname.new("#{Pathname.new(__FILE__).realpath.dirname}/../../bin").realpath
    end
    module_function :bindir

    # Copy image files under from_dir to to_dir recursively
    # ==== Args
    # from_dir :: path to the directory which has image files to be copied
    # to_dir :: path to the directory to which the image files are copied
    # options :: used to specify optional operations during copy
    # ==== Returns
    # list of image files
    # ==== Options
    # :convert :: Conversion rule
    # ==== Examples
    #
    #   copy_images_to_dir("/path/to/foo", "/path/to/bar", :convert => {:eps => :png})
    #
    # Image files are copied recursively, and each '.eps' file is converted into '.eps.png'
    #

    def copy_images_to_dir(from_dir, to_dir, options = {})
      image_files = []

      Dir.open(from_dir) do |dir|
        dir.each do |fname|
          next if fname =~ /^\./
          if FileTest.directory?("#{from_dir}/#{fname}")
            image_files += copy_images_to_dir("#{from_dir}/#{fname}", "#{to_dir}/#{fname}", options)
          else
            FileUtils.mkdir_p(to_dir) unless File.exist?(to_dir)

            is_converted = false
            (options[:convert] || {}).each do |orig_type, conv_type|
              next unless /\.#{orig_type}$/ =~ fname
              is_converted = system("convert #{from_dir}/#{fname} #{to_dir}/#{fname}.#{conv_type}")
              image_files << "#{from_dir}/#{fname}.#{conv_type}"
            end

            exts = options[:exts] || %w[png gif jpg jpeg svg pdf eps ai tif psd]
            exts_str = exts.join('|')
            if !is_converted && fname =~ /\.(#{exts_str})$/i
              FileUtils.cp("#{from_dir}/#{fname}", to_dir)
              image_files << "#{from_dir}/#{fname}"
            end
          end
        end
      end

      image_files
    end
    module_function :copy_images_to_dir

    def cleanup_mathimg(path = '_review_math')
      math_dir = "./#{@config['imagedir']}/#{path}"
      if @config['imgmath'] && Dir.exist?(math_dir)
        FileUtils.rm_rf(math_dir)
      end
    end

    def default_imgmath_preamble
      <<-EOB
\\documentclass[uplatex,a3paper,landscape]{jsarticle}
\\usepackage[deluxe,uplatex]{otf}
\\usepackage[T1]{fontenc}
\\usepackage{textcomp}
\\usepackage{lmodern}
\\usepackage[dvipdfmx]{graphicx}
\\usepackage[dvipdfmx,table]{xcolor}
\\usepackage[utf8]{inputenc}
\\usepackage{ascmac}
\\usepackage{float}
\\usepackage{alltt}
\\usepackage{amsmath}
\\usepackage{amssymb}
\\usepackage{amsfonts}
\\usepackage{anyfontsize}
\\usepackage{bm}
\\pagestyle{empty}
% \\setpaperwidth{1000mm}
    EOB
    end

    def make_math_images(math_dir)
      fontsize = @config['imgmath_options']['fontsize'].to_f
      lineheight = @config['imgmath_options']['lineheight'].to_f

      texsrc = default_imgmath_preamble
      if @config['imgmath_options']['preamble_file'] && File.readable?(@config['imgmath_options']['preamble_file'])
        texsrc = File.read(@config['imgmath_options']['preamble_file'])
      end

      texsrc << <<-EOB
\\begin{document}
\\fontsize{#{fontsize}}{#{lineheight}}\\selectfont
\\input{__IMGMATH_BODY__}
\\end{document}
EOB

      hashes = File.readlines(File.join(math_dir, '__IMGMATH_BODY__.map')).sort.uniq
      File.write(File.join(math_dir, '__IMGMATH_BODY__.map'), hashes.join)

      File.open(File.join(math_dir, '__IMGMATH_BODY__.tex'), 'w') do |f|
        File.open(File.join(math_dir, '__IMGMATH_BODY__.map')) do |map|
          map.each_line do |l|
            l.chomp!
            f.puts "% #{l}"
            f.puts File.read(File.join(math_dir, "__IMGMATH_BODY__#{l}.tex"))
            File.unlink(File.join(math_dir, "__IMGMATH_BODY__#{l}.tex"))
            f.puts '\\clearpage'
            f.puts
          end
        end
      end

      math_dir = File.realpath(math_dir)
      Dir.mktmpdir do |tmpdir|
        FileUtils.cp([File.join(math_dir, '__IMGMATH_BODY__.tex'),
                      File.join(math_dir, '__IMGMATH_BODY__.map')],
                     tmpdir)
        tex_path = File.join(tmpdir, '__IMGMATH__.tex')
        File.write(tex_path, texsrc)

        begin
          case @config['imgmath_options']['converter']
          when 'pdfcrop'
            make_math_images_pdfcrop(tmpdir, tex_path, math_dir)
          when 'dvipng'
            make_math_images_dvipng(tmpdir, tex_path, math_dir)
          else
            error "unknown math converter error. imgmath_options/converter parameter should be 'pdfcrop' or 'dvipng'."
          end
        rescue CompileError
          FileUtils.cp([tex_path,
                        File.join(File.dirname(tex_path), '__IMGMATH__.log')],
                       math_dir)
          error "LaTeX math compile error. See #{math_dir}/__IMGMATH__.log for details."
        end
      end
      FileUtils.rm_f([File.join(math_dir, '__IMGMATH_BODY__.tex'),
                      File.join(math_dir, '__IMGMATH_BODY__.map')])
    end

    def make_math_images_pdfcrop(dir, tex_path, math_dir)
      # rubocop:disable Metrics/BlockLength
      Dir.chdir(dir) do
        dvi_path = '__IMGMATH__.dvi'
        pdf_path = '__IMGMATH__.pdf'
        out, status = Open3.capture2e(*[@config['texcommand'], @config['texoptions'].shellsplit, tex_path].flatten.compact)
        if !status.success? || (!File.exist?(dvi_path) && !File.exist?(pdf_path))
          raise CompileError
        end
        if File.exist?(dvi_path)
          out, status = Open3.capture2e(*[@config['dvicommand'], @config['dvioptions'].shellsplit, dvi_path].flatten.compact)
          if !status.success? || !File.exist?(pdf_path)
            warn "error in #{@config['dvicommand']}. Error log:\n#{out}"
            raise CompileError
          end
        end
        args = @config['imgmath_options']['pdfcrop_cmd'].shellsplit
        args.map! do |m|<commit_msg>prepare for test<commit_after># Copyright (c) 2012-2020 <NAME>, <NAME>
#
# This program is free software.
# You can distribute or modify this program under the terms of
# the GNU LGPL, Lesser General Public License version 2.1.
# For details of the GNU LGPL, see the file "COPYING".
#
require 'pathname'
require 'fileutils'
require 'yaml'
require 'shellwords'

module ReVIEW
  module MakerHelper
    # Return review/bin directory
    def bindir
      Pathname.new("#{Pathname.new(__FILE__).realpath.dirname}/../../bin").realpath
    end
    module_function :bindir

    # Copy image files under from_dir to to_dir recursively
    # ==== Args
    # from_dir :: path to the directory which has image files to be copied
    # to_dir :: path to the directory to which the image files are copied
    # options :: used to specify optional operations during copy
    # ==== Returns
    # list of image files
    # ==== Options
    # :convert :: Conversion rule
    # ==== Examples
    #
    #   copy_images_to_dir("/path/to/foo", "/path/to/bar", :convert => {:eps => :png})
    #
    # Image files are copied recursively, and each '.eps' file is converted into '.eps.png'
    #

    def copy_images_to_dir(from_dir, to_dir, options = {})
      image_files = []

      Dir.open(from_dir) do |dir|
        dir.each do |fname|
          next if fname =~ /^\./
          if FileTest.directory?("#{from_dir}/#{fname}")
            image_files += copy_images_to_dir("#{from_dir}/#{fname}", "#{to_dir}/#{fname}", options)
          else
            FileUtils.mkdir_p(to_dir) unless File.exist?(to_dir)

            is_converted = false
            (options[:convert] || {}).each do |orig_type, conv_type|
              next unless /\.#{orig_type}$/ =~ fname
              is_converted = system("convert #{from_dir}/#{fname} #{to_dir}/#{fname}.#{conv_type}")
              image_files << "#{from_dir}/#{fname}.#{conv_type}"
            end

            exts = options[:exts] || %w[png gif jpg jpeg svg pdf eps ai tif psd]
            exts_str = exts.join('|')
            if !is_converted && fname =~ /\.(#{exts_str})$/i
              FileUtils.cp("#{from_dir}/#{fname}", to_dir)
              image_files << "#{from_dir}/#{fname}"
            end
          end
        end
      end

      image_files
    end
    module_function :copy_images_to_dir

    def cleanup_mathimg(path = '_review_math')
      math_dir = "./#{@config['imagedir']}/#{path}"
      if @config['imgmath'] && Dir.exist?(math_dir)
        FileUtils.rm_rf(math_dir)
      end
    end

    def default_imgmath_preamble
      <<-EOB
\\documentclass[uplatex,a3paper,landscape]{jsarticle}
\\usepackage[deluxe,uplatex]{otf}
\\usepackage[T1]{fontenc}
\\usepackage{textcomp}
\\usepackage{lmodern}
\\usepackage[dvipdfmx]{graphicx}
\\usepackage[dvipdfmx,table]{xcolor}
\\usepackage[utf8]{inputenc}
\\usepackage{ascmac}
\\usepackage{float}
\\usepackage{alltt}
\\usepackage{amsmath}
\\usepackage{amssymb}
\\usepackage{amsfonts}
\\usepackage{anyfontsize}
\\usepackage{bm}
\\pagestyle{empty}
% \\setpaperwidth{1000mm}
    EOB
    end

    def make_math_images(math_dir)
      fontsize = @config['imgmath_options']['fontsize'].to_f
      lineheight = @config['imgmath_options']['lineheight'].to_f

      texsrc = default_imgmath_preamble
      if @config['imgmath_options']['preamble_file'] && File.readable?(@config['imgmath_options']['preamble_file'])
        texsrc = File.read(@config['imgmath_options']['preamble_file'])
      end

      texsrc << <<-EOB
\\begin{document}
\\fontsize{#{fontsize}}{#{lineheight}}\\selectfont
\\input{__IMGMATH_BODY__}
\\end{document}
EOB

      hashes = File.readlines(File.join(math_dir, '__IMGMATH_BODY__.map')).sort.uniq
      File.write(File.join(math_dir, '__IMGMATH_BODY__.map'), hashes.join)

      File.open(File.join(math_dir, '__IMGMATH_BODY__.tex'), 'w') do |f|
        File.open(File.join(math_dir, '__IMGMATH_BODY__.map')) do |map|
          map.each_line do |l|
            l.chomp!
            f.puts "% #{l}"
            f.puts File.read(File.join(math_dir, "__IMGMATH_BODY__#{l}.tex"))
            File.unlink(File.join(math_dir, "__IMGMATH_BODY__#{l}.tex"))
            f.puts '\\clearpage'
            f.puts
          end
        end
      end

      math_dir = File.realpath(math_dir)
      Dir.mktmpdir do |tmpdir|
        FileUtils.cp([File.join(math_dir, '__IMGMATH_BODY__.tex'),
                      File.join(math_dir, '__IMGMATH_BODY__.map')],
                     tmpdir)
        tex_path = File.join(tmpdir, '__IMGMATH__.tex')
        File.write(tex_path, texsrc)

        begin
          case @config['imgmath_options']['converter']
          when 'pdfcrop'
            make_math_images_pdfcrop(tmpdir, tex_path, math_dir)
          when 'dvipng'
            make_math_images_dvipng(tmpdir, tex_path, math_dir)
          else
            error "unknown math converter error. imgmath_options/converter parameter should be 'pdfcrop' or 'dvipng'."
          end
        rescue CompileError
          FileUtils.cp([tex_path,
                        File.join(File.dirname(tex_path), '__IMGMATH__.log')],
                       math_dir)
          error "LaTeX math compile error. See #{math_dir}/__IMGMATH__.log for details."
        end
      end
      FileUtils.rm_f([File.join(math_dir, '__IMGMATH_BODY__.tex'),
                      File.join(math_dir, '__IMGMATH_BODY__.map')])
    end
    module_function :make_math_images

    def make_math_images_pdfcrop(dir, tex_path, math_dir)
      # rubocop:disable Metrics/BlockLength
      Dir.chdir(dir) do
        dvi_path = '__IMGMATH__.dvi'
        pdf_path = '__IMGMATH__.pdf'
        out, status = Open3.capture2e(*[@config['texcommand'], @config['texoptions'].shellsplit, tex_path].flatten.compact)
        if !status.success? || (!File.exist?(dvi_path) && !File.exist?(pdf_path))
          raise CompileError
        end
        if File.exist?(dvi_path)
          out, status = Open3.capture2e(*[@config['dvicommand'], @config['dvioptions'].shellsplit, dvi_path].flatten.compact)
          if !status.success? || !File.exist?(pdf_path)
            warn "error in #{@config['dvicommand']}. Error log:\n#{out}"
            raise CompileError
          end
        end
        args = @config['imgmath_options']['pdfcrop_cmd'].shellsplit
        args.map! do |m|
{'fraction_non_alphanumeric': 0.13945551234224937, 'fraction_numerical': 0.0033335979045956026, 'mean_word_length': 2.6595991867557363, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7556608', 'n_tokens_mistral': 4248, 'n_tokens_neox': 4150, 'n_words': 809}