<commit_before>module CdmMigrator
	class CdmController < ApplicationController
		helper_method :default_page_title, :admin_host?, :available_translations, :available_works
		layout 'dashboard' if Hyrax
		require 'csv'

		before_action :load_yaml
		before_action :set_exclusive_fields, only: [:generate, :mappings]
		skip_before_action :verify_authenticity_token

		def generate
			@h_to_c = {}
			@c_to_h = {}
			params[:mappings].each do |key, mapping|
				if !mapping['hydra'].empty?
					@c_to_h[mapping['cdm']] = mapping['hydra']
					@h_to_c[mapping['hydra']] ||= []
					@h_to_c[mapping['hydra']] << mapping['cdm']
				elsif !mapping['hydrac'].empty?
					@c_to_h[mapping['cdm']] = mapping['hydrac']
					@h_to_c[mapping['hydrac']] ||= []
					@h_to_c[mapping['hydrac']] << mapping['cdm']
				end
			end
			json = JSON.parse(Net::HTTP.get_response(URI.parse("#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/0/0/0/0/0/1/0/json")).body)
			total_recs = json['pager']['total'].to_i
			if total_recs > 1024
				start = 1
				records = []
				[0..(total_recs/1024)].each do |index|
					start = (index*1024) + 1
					json = JSON.parse(Net::HTTP.get_response(URI.parse("http://#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/#{start}/0/0/0/0/1/0/json")).body)
					records << json['records'].map { |rec| [rec['pointer'], rec['filetype']] }
				end
			else
				records = json['records'].map { |rec| [rec['pointer'], rec['filetype']] }
			end
			headers = ::CSV.generate_line (['object_type','url']+@terms+@work_only)<commit_msg>fixed issue with large collections failing to map<commit_after>module CdmMigrator
	class CdmController < ApplicationController
		helper_method :default_page_title, :admin_host?, :available_translations, :available_works
		layout 'dashboard' if Hyrax
		require 'csv'

		before_action :load_yaml
		before_action :set_exclusive_fields, only: [:generate, :mappings]
		skip_before_action :verify_authenticity_token

		def generate
			@h_to_c = {}
			@c_to_h = {}
			params[:mappings].each do |key, mapping|
				if !mapping['hydra'].empty?
					@c_to_h[mapping['cdm']] = mapping['hydra']
					@h_to_c[mapping['hydra']] ||= []
					@h_to_c[mapping['hydra']] << mapping['cdm']
				elsif !mapping['hydrac'].empty?
					@c_to_h[mapping['cdm']] = mapping['hydrac']
					@h_to_c[mapping['hydrac']] ||= []
					@h_to_c[mapping['hydrac']] << mapping['cdm']
				end
			end
			json = JSON.parse(Net::HTTP.get_response(URI.parse("#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/0/0/0/0/0/1/0/json")).body)
			total_recs = json['pager']['total'].to_i
			if total_recs > 1024
				start = 1
				records = []
				(0..(total_recs/1024)).each do |index|
					start = (index*1024) + 1
					json = JSON.parse(Net::HTTP.get_response(URI.parse("#{@cdm_url}:#{@cdm_port}/dmwebservices/index.php?q=dmQuery#{params[:collection]}/0/0/filetype/1024/#{start}/0/0/0/0/1/0/json")).body)
					records << json['records'].map { |rec| [rec['pointer'], rec['filetype']] }
				end
			else
				records = json['records'].map { |rec| [rec['pointer'], rec['filetype']] }
			end
			headers = ::CSV.generate_line (['object_type','url']+@terms+@work_only)
{'fraction_non_alphanumeric': 0.19318527532704594, 'fraction_numerical': 0.024338302403407362, 'mean_word_length': 4.688581314878893, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6879827', 'n_tokens_mistral': 1469, 'n_tokens_neox': 1239, 'n_words': 172}