<commit_before>require 'contentdm_translator'
class ExportController < ApplicationController
  require 'zip'
  include CollectionHelper,ExportHelper

  # no layout if xhr request
  layout Proc.new { |controller| controller.request.xhr? ? false : nil } #, :only => [:update, :update_profile]

  README = []

  def index
    @collection = Collection.friendly.find(params[:collection_id])
    #check if there are any translated works in the collection
    if @collection.works.where(supports_translation: true).exists?
      @header = "Translated"
    else
      @header = "Transcribed"
    end

    @works = @collection.works.includes(:work_statistic).paginate(page: params[:page], per_page: 15)

    @table_export = @collection.works.joins(:table_cells).where.not(table_cells: {work_id: nil}).distinct
  end

  def show
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def text
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def transcript
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def translation
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def tei
    params[:format] = 'xml'# if params[:format].blank?

    @context = ExportContext.new

    @user_contributions =
      User.find_by_sql("SELECT  user_id user_id,
                                users.print_name print_name,
                                count(*) edit_count,
                                min(page_versions.created_on) first_edit,
                                max(page_versions.created_on) last_edit
                        FROM    page_versions
                        INNER JOIN pages
                            ON page_versions.page_id = pages.id
                        INNER JOIN users
                            ON page_versions.user_id = users.id
                        WHERE pages.work_id = #{@work.id}
                          AND page_versions.transcription IS NOT NULL
                        GROUP BY user_id
                        ORDER BY count(*) DESC")

    @work_versions = PageVersion.joins(:page).where(['pages.work_id = ?', @work.id]).order("work_version DESC").includes(:page).all

    @all_articles = @work.articles

    @person_articles = @all_articles.joins(:categories).where(categories: {title: 'People'})
    @place_articles = @all_articles.joins(:categories).where(categories: {title: 'Places'})
    @other_articles = @all_articles.joins(:categories).where.not(categories: {title: 'People'})
                      .where.not(categories: {title: 'Places'})
    
    ### Catch the rendered Work for post-processing
    xml = render_to_string :layout => false, :template => "export/tei.html.erb"

    # Render the post-processed
    render :text => post_process_xml(xml, @work), :content_type => "application/xml"
  end

  def subject_csv
    send_data(@collection.export_subjects_as_csv,
              :filename => "fromthepage_subjects_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
              :type => "application/csv")
    cookies['download_finished'] = 'true'
  end
  
  def table_csv
    send_data(export_tables_as_csv(@work),
              :filename => "fromthepage_tables_export_#{@work.id}_#{Time.now.utc.iso8601}.csv",
              :type => "application/csv")
    cookies['download_finished'] = 'true'
    
  end

  def desc_to_file(text:, dirname:, out:)
    README << text

    path = File.join dirname, 'README.txt'
    out.put_next_entry path

    README.each do |desc|
      out.write desc
    end
  end

  def export_plaintext_transcript(name:, dirname:, out:)
    path = File.join dirname, 'plaintext', "#{name}_transcript.txt"

    out.put_next_entry path

    case name
    when "verbatim"
      out.write @work.verbatim_transcription_plaintext
      desc_to_file(text: "plaintext/verbatim_transcript.txt - file containing the per-document verbatim plaintext export of the transcript.\n", dirname: dirname, out: out)
    when "emended"
      out.write @work.emended_transcription_plaintext
      desc_to_file(text: "plaintext/emended_transcript.txt - file containing the per-document emended plaintext export of the transcript.\n", dirname: dirname, out: out)
    when "searchable"
      out.write @work.searchable_plaintext
      desc_to_file(text: "plaintext/searchable_transcript.txt - file containing the per-document searchable plaintext export of the transcript.\n", dirname: dirname, out: out)
    end
  end

  def export_plaintext_translation(name:, dirname:, out:)
    path = File.join dirname, 'plaintext', "#{name}_translation.txt"

    out.put_next_entry path

    case name
    when "verbatim"
      out.write @work.verbatim_translation_plaintext
      desc_to_file(text: "plaintext/verbatim_translation.txt - file containing the per-document verbatim plaintext export of the translation if it is present.\n", dirname: dirname, out: out)
    when "emended"
      out.write @work.emended_translation_plaintext
      desc_to_file(text: "plaintext/emended_translation.txt - file containing the per-document emended plaintext export of the translation if it is present.\n", dirname: dirname, out: out)
    end
  end

  def export_plaintext_transcript_pages(name:, dirname:, out:, page:)
    path = File.join dirname, 'plaintext', "#{name}_transcript_pages", "#{page.title}.txt"

    out.put_next_entry path

    case name
    when "verbatim"
      out.write page.verbatim_transcription_plaintext
      desc_to_file(text: "plaintext/verbatim_transcript_pages/#{page.title}.txt - files containing per-page verbatim plaintext export.\n", dirname: dirname, out: out)
    when "emended"
      out.write page.emended_transcription_plaintext
      desc_to_file(text: "plaintext/emended_transcript_pages/#{page.title}.txt - files containing per-page verbatim plaintext export.\n", dirname: dirname, out: out)
    end
  end

  def export_plaintext_translation_pages(name:, dirname:, out:, page:)
    path = File.join dirname, 'plaintext', "#{name}_translation_pages", "#{page.title}.txt"

    out.put_next_entry path

    case name
    when "verbatim"
      out.write page.verbatim_translation_plaintext
      desc_to_file(text: "plaintext/verbatim_translation_pages/#{page.title}.txt - files containing per-page verbatim plaintext export.\n", dirname: dirname, out: out)
    when "emended"
      out.write page.emended_translation_plaintext
      desc_to_file(text: "plaintext/emended_translation_pages/#{page.title}.txt - files containing per-page verbatim plaintext export.\n", dirname: dirname, out: out)
    end<commit_msg>export: refactor export method calls<commit_after>require 'contentdm_translator'
class ExportController < ApplicationController
  require 'zip'
  include CollectionHelper,ExportHelper

  # no layout if xhr request
  layout Proc.new { |controller| controller.request.xhr? ? false : nil } #, :only => [:update, :update_profile]

  README = []

  def index
    @collection = Collection.friendly.find(params[:collection_id])
    #check if there are any translated works in the collection
    if @collection.works.where(supports_translation: true).exists?
      @header = "Translated"
    else
      @header = "Transcribed"
    end

    @works = @collection.works.includes(:work_statistic).paginate(page: params[:page], per_page: 15)

    @table_export = @collection.works.joins(:table_cells).where.not(table_cells: {work_id: nil}).distinct
  end

  def show
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def text
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def transcript
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def translation
    @work = Work.includes(pages: [:notes, {page_versions: :user}]).find_by(id: params[:work_id])
    render :layout => false
  end

  def tei
    params[:format] = 'xml'# if params[:format].blank?

    @context = ExportContext.new

    @user_contributions =
      User.find_by_sql("SELECT  user_id user_id,
                                users.print_name print_name,
                                count(*) edit_count,
                                min(page_versions.created_on) first_edit,
                                max(page_versions.created_on) last_edit
                        FROM    page_versions
                        INNER JOIN pages
                            ON page_versions.page_id = pages.id
                        INNER JOIN users
                            ON page_versions.user_id = users.id
                        WHERE pages.work_id = #{@work.id}
                          AND page_versions.transcription IS NOT NULL
                        GROUP BY user_id
                        ORDER BY count(*) DESC")

    @work_versions = PageVersion.joins(:page).where(['pages.work_id = ?', @work.id]).order("work_version DESC").includes(:page).all

    @all_articles = @work.articles

    @person_articles = @all_articles.joins(:categories).where(categories: {title: 'People'})
    @place_articles = @all_articles.joins(:categories).where(categories: {title: 'Places'})
    @other_articles = @all_articles.joins(:categories).where.not(categories: {title: 'People'})
                      .where.not(categories: {title: 'Places'})
    
    ### Catch the rendered Work for post-processing
    xml = render_to_string :layout => false, :template => "export/tei.html.erb"

    # Render the post-processed
    render :text => post_process_xml(xml, @work), :content_type => "application/xml"
  end

  def subject_csv
    send_data(@collection.export_subjects_as_csv,
              :filename => "fromthepage_subjects_export_#{@collection.id}_#{Time.now.utc.iso8601}.csv",
              :type => "application/csv")
    cookies['download_finished'] = 'true'
  end
  
  def table_csv
    send_data(export_tables_as_csv(@work),
              :filename => "fromthepage_tables_export_#{@work.id}_#{Time.now.utc.iso8601}.csv",
              :type => "application/csv")
    cookies['download_finished'] = 'true'
    
  end

  def desc_to_file(text:, dirname:, out:)
    README << text

    path = File.join dirname, 'README.txt'
    out.put_next_entry path

    README.each do |desc|
      out.write desc
    end
  end

  def export_plaintext_transcript(name:, dirname:, out:)
    path = File.join dirname, 'plaintext', "#{name}_transcript.txt"

    out.put_next_entry path

    case name
    when "verbatim"
      out.write @work.verbatim_transcription_plaintext
      desc_to_file(text: "plaintext/verbatim_transcript.txt - file containing the per-document verbatim plaintext export of the transcript.\n", dirname: dirname, out: out)
    when "emended"
      out.write @work.emended_transcription_plaintext
      desc_to_file(text: "plaintext/emended_transcript.txt - file containing the per-document emended plaintext export of the transcript.\n", dirname: dirname, out: out)
    when "searchable"
      out.write @work.searchable_plaintext
      desc_to_file(text: "plaintext/searchable_transcript.txt - file containing the per-document searchable plaintext export of the transcript.\n", dirname: dirname, out: out)
    end
  end

  def export_plaintext_translation(name:, dirname:, out:)
    path = File.join dirname, 'plaintext', "#{name}_translation.txt"

    out.put_next_entry path

    out.write @work.public_send "#{name}_translation_plaintext"
    desc_to_file(text: "plaintext/#{name}_translation.txt - file containing the per-document #{name} plaintext export of the translation if it is present.\n", dirname: dirname, out: out)
  end

  def export_plaintext_transcript_pages(name:, dirname:, out:, page:)
    path = File.join dirname, 'plaintext', "#{name}_transcript_pages", "#{page.title}.txt"

    out.put_next_entry path

    out.write page.public_send "#{name}_transcription_plaintext"
    desc_to_file(text: "plaintext/#{name}_transcript_pages/#{page.title}.txt - files containing per-page #{name} plaintext export.\n", dirname: dirname, out: out)
  end

  def export_plaintext_translation_pages(name:, dirname:, out:, page:)
    path = File.join dirname, 'plaintext', "#{name}_translation_pages", "#{page.title}.txt"

    out.put_next_entry path

    out.write page.public_send "#{name}_translation_plaintext"
    desc_to_file(text: "plaintext/#{name}_translation_pages/#{page.title}.txt - files containing per-page #{name} plaintext export.\n", dirname: dirname, out: out)
{'fraction_non_alphanumeric': 0.11451751306654186, 'fraction_numerical': 0.0015601841017240034, 'mean_word_length': 3.195026178010471, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1768916', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3851, 'n_words': 877}