<commit_before>require 'fileutils'
require 'thread'
require "net/anidbudp"
class Biribiri::Processor
	attr_accessor :log, :testmode, :animebase, :moviebase, :backlog_set
	attr_accessor :anidb_server, :anidb_port, :anidb_remoteport, :anidb_username, :anidb_password, :anidb_nat
	attr_accessor :renamer

	FILE_FFIELDS = [ :aid, :eid, :gid, :length, :quality, :video_resolution,
	                 :source, :sub_language, :dub_language, :video_codec,
	                 :audio_codec_list, :crc32, :state, :file_type ]

	FILE_FSTATES = {
		:crcok      => 1,
		:crcerr     => 2,
		:version2   => 4,
		:version3   => 8,
		:version4   => 16,
		:version5   => 32,
		:uncensored => 64,
		:censored   => 128
	}

	VERSION_MAP = {
		:version2 => 2,
		:version3 => 3,
		:version4 => 4,
		:version5 => 5
	}

	class Renamer
		def self.rename(file)
			raise NotImplementedError
		end
	end

	def setup
		# setup queues
		@ed2k_queue = Queue.new
		@info_queue = Queue.new
		@process_queue = Queue.new

		if @testmode
			Logger.log.info("[#] Running in test mode. Files won't be renamed.")
		end

		# setup ed2k hash worker
		@ed2k_worker = Thread.new do
			# Go forever while we have stuff to process
			while true
				Logger.log.debug("[H] Waiting for next file to hash")
				file = @ed2k_queue.pop
				break unless file
				Logger.log.debug("[H] Hashing #{File.basename(file)}")
				size, hash = Net::AniDBUDP.ed2k_file_hash(file)
				@info_queue << { :size => size, :hash => hash, :file => file }
				Logger.log.info("[H] #{File.basename(file)} (H: #{hash}, S: #{size})")
			end

			# Tell the next processor that we're done sending it things
			@info_queue << nil
		end

		@info_worker = Thread.new do
			anidb = Net::AniDBUDP.new(@anidb_server, @anidb_port, @anidb_remoteport)
			anidb.connect(@anidb_username, @anidb_password, @anidb_nat)

			while true
				Logger.log.debug("[I] Waiting for next file to get info")
				src = @info_queue.pop
				break unless src
				Logger.log.debug("[I] Searching #{File.basename(src[:file])}")
				file = anidb.search_file(File.basename(src[:file]), src[:size], src[:hash], FILE_FFIELDS)
				if file.nil?
          Logger.log.warn("[I] #{src[:file]} can't be found. ed2k://|file|#{src[:file]}|#{src[:size]}|#{src[:hash]}|/")
					next
				end
				Logger.log.info("[I] #{File.basename(src[:file])} => #{file[:anime][:romaji_name]} (EP: #{file[:anime][:epno]}, FID: #{file[:fid]}, AID: #{file[:file][:aid]})")


				# Extract the states variable into something more sane
				# <NAME> please bear my children
				state_keys = FILE_FSTATES.reject { |k,v| ((file[:file][:state].to_i || 0) & v).zero? }.keys
				file[:file][:state_keys] = state_keys

				file[:file][:crcstatus] = (state_keys & [:crcok, :crcerr]).first
				file[:file][:censoredstatus] = (state_keys & [:uncensored, :censored]).first

				# This is kinda gross, but it works.
				# Coded to get the maximum version provided in case of brain damage
				version = (state_keys & [:version2, :version3, :version4, :version5]).map { |i| VERSION_MAP[i] }.max
				file[:file][:version] = version || 1

				# Uncomment for debugging
				#pp file
				@process_queue << {:src => src, :file => file}
				Logger.log.debug("[I] Added #{File.basename(src[:file])} to process queue")
			end

			anidb.logout<commit_msg>Changed ed2k links to use the base name of the file. Fixed an indent issue.<commit_after>require 'fileutils'
require 'thread'
require "net/anidbudp"
class Biribiri::Processor
	attr_accessor :log, :testmode, :animebase, :moviebase, :backlog_set
	attr_accessor :anidb_server, :anidb_port, :anidb_remoteport, :anidb_username, :anidb_password, :anidb_nat
	attr_accessor :renamer

	FILE_FFIELDS = [ :aid, :eid, :gid, :length, :quality, :video_resolution,
	                 :source, :sub_language, :dub_language, :video_codec,
	                 :audio_codec_list, :crc32, :state, :file_type ]

	FILE_FSTATES = {
		:crcok      => 1,
		:crcerr     => 2,
		:version2   => 4,
		:version3   => 8,
		:version4   => 16,
		:version5   => 32,
		:uncensored => 64,
		:censored   => 128
	}

	VERSION_MAP = {
		:version2 => 2,
		:version3 => 3,
		:version4 => 4,
		:version5 => 5
	}

	class Renamer
		def self.rename(file)
			raise NotImplementedError
		end
	end

	def setup
		# setup queues
		@ed2k_queue = Queue.new
		@info_queue = Queue.new
		@process_queue = Queue.new

		if @testmode
			Logger.log.info("[#] Running in test mode. Files won't be renamed.")
		end

		# setup ed2k hash worker
		@ed2k_worker = Thread.new do
			# Go forever while we have stuff to process
			while true
				Logger.log.debug("[H] Waiting for next file to hash")
				file = @ed2k_queue.pop
				break unless file
				Logger.log.debug("[H] Hashing #{File.basename(file)}")
				size, hash = Net::AniDBUDP.ed2k_file_hash(file)
				@info_queue << { :size => size, :hash => hash, :file => file }
				Logger.log.info("[H] #{File.basename(file)} (H: #{hash}, S: #{size})")
			end

			# Tell the next processor that we're done sending it things
			@info_queue << nil
		end

		@info_worker = Thread.new do
			anidb = Net::AniDBUDP.new(@anidb_server, @anidb_port, @anidb_remoteport)
			anidb.connect(@anidb_username, @anidb_password, @anidb_nat)

			while true
				Logger.log.debug("[I] Waiting for next file to get info")
				src = @info_queue.pop
				break unless src
				Logger.log.debug("[I] Searching #{File.basename(src[:file])}")
				file = anidb.search_file(File.basename(src[:file]), src[:size], src[:hash], FILE_FFIELDS)
				if file.nil?
          			Logger.log.warn("[I] #{src[:file]} can't be found. ed2k://|file|#{File.basename(src[:file])}|#{src[:size]}|#{src[:hash]}|/")
					next
				end
				Logger.log.info("[I] #{File.basename(src[:file])} => #{file[:anime][:romaji_name]} (EP: #{file[:anime][:epno]}, FID: #{file[:fid]}, AID: #{file[:file][:aid]})")


				# Extract the states variable into something more sane
				# <NAME> please bear my children
				state_keys = FILE_FSTATES.reject { |k,v| ((file[:file][:state].to_i || 0) & v).zero? }.keys
				file[:file][:state_keys] = state_keys

				file[:file][:crcstatus] = (state_keys & [:crcok, :crcerr]).first
				file[:file][:censoredstatus] = (state_keys & [:uncensored, :censored]).first

				# This is kinda gross, but it works.
				# Coded to get the maximum version provided in case of brain damage
				version = (state_keys & [:version2, :version3, :version4, :version5]).map { |i| VERSION_MAP[i] }.max
				file[:file][:version] = version || 1

				# Uncomment for debugging
				#pp file
				@process_queue << {:src => src, :file => file}
				Logger.log.debug("[I] Added #{File.basename(src[:file])} to process queue")
			end

			anidb.logout
{'fraction_non_alphanumeric': 0.1600541027953111, 'fraction_numerical': 0.011872557859933875, 'mean_word_length': 3.670175438596491, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042486', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2475, 'n_words': 587}