<commit_before>require 't13n/core_ext/string'
require 'xpathquery'

class Dictionary < ActiveRecord::Base
	EXIST_REST_ENDPOINT = 'http://localhost:8080/exist/rest/db'

	ENTRY_ID_EQUALS = '@xml:id = "lemma-%{entry_id}"'
	IS_DICT_ENTRY = 'self::tei:entry or self::tei:re'
	ORTH_EQUALS = './tei:form/tei:orth/text() = "%{term}"' # FIXME: escape parameters

	DICT_ENTRIES = "/tei:TEI/tei:text/tei:body//*[#{IS_DICT_ENTRY}]"
	DICT_SCANS = '/tei:TEI/tei:facsimile/tei:graphic'

	NS = {
		'tei' => 'http://www.tei-c.org/ns/1.0',
	}

	after_initialize :setup_query_engine

	def setup_query_engine
		dict_path = self.content_path
		@dict_db = EXIST_REST_ENDPOINT + '/' + "dict/#{dict_path}"
		@query_engine = XPathQuery::Exist.new(@dict_db, :logger => Rails.logger)
	end

	def xpathquery(query, params = {})
		@query_engine.query(query, params, NS)
	end

	def header
		# FIXME: declare stale after X seconds and compare cache
		@header ||= xpathquery('/tei:TEI/tei:teiHeader').first
		return @header
	end

	def backmatter
		# FIXME: use a common cache system with `header`
		@backmatter ||= xpathquery('/tei:TEI/tei:text/tei:back').first
		return @backmatter
	end

	def backmatter_item(id)
		backmatter_complete = backmatter
		if !backmatter_complete.is_a? Nokogiri::XML::Element
			return []
		end

		return backmatter_complete.xpath("//*[@xml:id = '#{id}']", NS)
	end

	def matches(tei_entries)
		# extract the top level <entry> or <re> elements
		entries = tei_entries.to_a.map { |tei_entry| entry_for_lemma(tei_entry) }
		return entries
	end

	def lemmas
		# FIXME: load only tei:form, not complete tei:(entry|re)
		tei_entries = all_entries.to_a

		entries = []

		tei_entries.each do |tei_entry|
			tei_orth = tei_entry.at("./tei:form/tei:orth", NS)
			tei_words = [tei_orth]

			transliterations = transliterations_for_words(tei_words)

			entries << {
				:entry => tei_entry,
				:transliterations => transliterations,
				:dict => publication_info,
			}
		end

		return entries
	end

	def lemmas_count
		query = "count(#{DICT_ENTRIES})"
		count = xpathquery(query).first

		return count
	end

	def lemma(entry_id, script)
		query = "#{DICT_ENTRIES}[#{ENTRY_ID_EQUALS}]"
		params = { :entry_id => entry_id }

		results = xpathquery(query, params)
		if results.empty?
			raise "Lemma not found"
		end

		tei_entry = results.first
		entry = entry_for_lemma(tei_entry)

		return entry
	end

	def entry_for_lemma(tei_entry)
		tei_words = tei_words_inside_tei_entry(tei_entry)
		transliterations = transliterations_for_words(tei_words)

		tei_refs = tei_refs_inside_tei_entry(tei_entry)
		references = references_for_refs(tei_refs)

		entry = {
			:entry => tei_entry,
			:transliterations => transliterations,
			:references => references,
			:dict => publication_info,
		}

		return entry
	end

	def transliterations_for_words(tei_words)
		transliterations = tei_words.map do |tei_word|
			word = tei_word.text # FIXME: include sub-tags?
			word.strip! # FIXME
			word.gsub!("\u221a", '!') # FIXME: deal with root char
			word.gsub!(/[0-9]/, '!') # FIXME: deal with digits

			raw_lang = tei_word.attr('xml:lang')

			transliterations = transliterations_for_word(word, raw_lang)

			next [word, transliterations_for_word(word, raw_lang)]
		end

		return transliterations.to_h
	end

	def tei_words_inside_tei_entry(tei_entry)
		tei_words = tei_entry.xpath("./*[not(self::tei:re)]//*[self::tei:orth or self::tei:hyph or self::tei:w or self::tei:m][(@xml:lang = 'san-Latn-x-SLP1') or (@xml:lang = 'san-Latn-x-SLP1-headword2')]", NS)
		return tei_words
	end

	def transliterations_for_word(word, lang_tag)
		mETHOD_FOR_LANG = {
			'san-Latn-x-SLP1' => :slp1,
			'san-Latn-x-SLP1-headword2' => :slp1_headword2
		}

		method = mETHOD_FOR_LANG[lang_tag]

		case lang_tag
		when 'san-Latn-x-SLP1', 'san-Latn-x-SLP1-headword2'
			native_script = word.transliterate(:Deva, :method => method)

			# TODO: only generate the needed transliterations
			transliterations = {
				'san-Deva' => native_script,
				'san-Latn-x-SLP1' => native_script.transliterate(:Latn, :method => :slp1),
				'san-Latn-x-ISO15919' => native_script.transliterate(:Latn, :method => :iso15919),
			}

			return transliterations
		else
			raise "Entry is stored in unknown language #{lang_tag.inspect}"
		end
	end

	def tei_refs_inside_tei_entry(tei_entry)
		tei_refs = tei_entry.xpath('.//tei:ref', NS)
		return tei_refs
	end

	def references_for_refs(tei_refs)
		targets = tei_refs.map { |tei_ref| tei_ref.attr('target') }
		ids = targets.map { |target| target.sub('#', '') }.uniq

		references = {}
		ids.each do |id|
			item = backmatter_item(id).first
			references[id] = item unless item.nil?

			nested_ref = item.at('.//tei:ref', NS) unless item.nil?
			if !nested_ref.nil?
				nested_id = nested_ref.attr('target').sub('#', '')
				nested_item = backmatter_item(nested_id)
				references[nested_id] = nested_item unless nested_item.nil?
			end
		end

		return references
	end


	def exact_matches(term)
		query = "#{DICT_ENTRIES}[#{ORTH_EQUALS}]"
		params = { :term => term }

		tei_matches = xpathquery(query, params)

		return matches(tei_matches)<commit_msg>Extract glyph referenced inside entries<commit_after>require 't13n/core_ext/string'
require 'xpathquery'

class Dictionary < ActiveRecord::Base
	EXIST_REST_ENDPOINT = 'http://localhost:8080/exist/rest/db'

	ENTRY_ID_EQUALS = '@xml:id = "lemma-%{entry_id}"'
	IS_DICT_ENTRY = 'self::tei:entry or self::tei:re'
	ORTH_EQUALS = './tei:form/tei:orth/text() = "%{term}"' # FIXME: escape parameters

	DICT_ENTRIES = "/tei:TEI/tei:text/tei:body//*[#{IS_DICT_ENTRY}]"
	DICT_SCANS = '/tei:TEI/tei:facsimile/tei:graphic'

	NS = {
		'tei' => 'http://www.tei-c.org/ns/1.0',
	}

	after_initialize :setup_query_engine

	def setup_query_engine
		dict_path = self.content_path
		@dict_db = EXIST_REST_ENDPOINT + '/' + "dict/#{dict_path}"
		@query_engine = XPathQuery::Exist.new(@dict_db, :logger => Rails.logger)
	end

	def xpathquery(query, params = {})
		@query_engine.query(query, params, NS)
	end

	def header
		# FIXME: declare stale after X seconds and compare cache
		@header ||= xpathquery('/tei:TEI/tei:teiHeader').first
		return @header
	end

	def header_item(id)
		# FIXME: merge with backmatter_item
		header_complete = header
		if !header_complete.is_a? Nokogiri::XML::Element
			return []
		end

		return header_complete.xpath("//*[@xml:id = '#{id}']", NS)
	end

	def backmatter
		# FIXME: use a common cache system with `header`
		@backmatter ||= xpathquery('/tei:TEI/tei:text/tei:back').first
		return @backmatter
	end

	def backmatter_item(id)
		backmatter_complete = backmatter
		if !backmatter_complete.is_a? Nokogiri::XML::Element
			return []
		end

		return backmatter_complete.xpath("//*[@xml:id = '#{id}']", NS)
	end

	def matches(tei_entries)
		# extract the top level <entry> or <re> elements
		entries = tei_entries.to_a.map { |tei_entry| entry_for_lemma(tei_entry) }
		return entries
	end

	def lemmas
		# FIXME: load only tei:form, not complete tei:(entry|re)
		tei_entries = all_entries.to_a

		entries = []

		tei_entries.each do |tei_entry|
			tei_orth = tei_entry.at("./tei:form/tei:orth", NS)
			tei_words = [tei_orth]

			transliterations = transliterations_for_words(tei_words)

			entries << {
				:entry => tei_entry,
				:transliterations => transliterations,
				:dict => publication_info,
			}
		end

		return entries
	end

	def lemmas_count
		query = "count(#{DICT_ENTRIES})"
		count = xpathquery(query).first

		return count
	end

	def lemma(entry_id, script)
		query = "#{DICT_ENTRIES}[#{ENTRY_ID_EQUALS}]"
		params = { :entry_id => entry_id }

		results = xpathquery(query, params)
		if results.empty?
			raise "Lemma not found"
		end

		tei_entry = results.first
		entry = entry_for_lemma(tei_entry)

		return entry
	end

	def entry_for_lemma(tei_entry)
		tei_words = tei_words_inside_tei_entry(tei_entry)
		transliterations = transliterations_for_words(tei_words)

		tei_refs = tei_refs_inside_tei_entry(tei_entry)
		references = references_for_refs(tei_refs)

		entry = {
			:entry => tei_entry,
			:transliterations => transliterations,
			:references => references,
			:dict => publication_info,
		}

		return entry
	end

	def transliterations_for_words(tei_words)
		transliterations = tei_words.map do |tei_word|
			word = tei_word.text # FIXME: include sub-tags?
			word.strip! # FIXME
			word.gsub!("\u221a", '!') # FIXME: deal with root char
			word.gsub!(/[0-9]/, '!') # FIXME: deal with digits

			raw_lang = tei_word.attr('xml:lang')

			transliterations = transliterations_for_word(word, raw_lang)

			next [word, transliterations_for_word(word, raw_lang)]
		end

		return transliterations.to_h
	end

	def tei_words_inside_tei_entry(tei_entry)
		tei_words = tei_entry.xpath("./*[not(self::tei:re)]//*[self::tei:orth or self::tei:hyph or self::tei:w or self::tei:m][(@xml:lang = 'san-Latn-x-SLP1') or (@xml:lang = 'san-Latn-x-SLP1-headword2')]", NS)
		return tei_words
	end

	def transliterations_for_word(word, lang_tag)
		mETHOD_FOR_LANG = {
			'san-Latn-x-SLP1' => :slp1,
			'san-Latn-x-SLP1-headword2' => :slp1_headword2
		}

		method = mETHOD_FOR_LANG[lang_tag]

		case lang_tag
		when 'san-Latn-x-SLP1', 'san-Latn-x-SLP1-headword2'
			native_script = word.transliterate(:Deva, :method => method)

			# TODO: only generate the needed transliterations
			transliterations = {
				'san-Deva' => native_script,
				'san-Latn-x-SLP1' => native_script.transliterate(:Latn, :method => :slp1),
				'san-Latn-x-ISO15919' => native_script.transliterate(:Latn, :method => :iso15919),
			}

			return transliterations
		else
			raise "Entry is stored in unknown language #{lang_tag.inspect}"
		end
	end

	def tei_refs_inside_tei_entry(tei_entry)
		tei_refs = tei_entry.xpath('.//tei:ref', NS)
		tei_refs += tei_entry.xpath('.//tei:g[@ref]', NS)
		return tei_refs
	end

	def references_for_refs(tei_refs)
		targets = tei_refs.map { |tei_ref| tei_ref.attr('target') || tei_ref.attr('ref') }
		ids = targets.map { |target| target.sub('#', '') }.uniq

		references = {}
		ids.each do |id|
			item = backmatter_item(id).first
			item ||= header_item(id).first
			references[id] = item unless item.nil?

			nested_ref = item.at('.//tei:ref', NS) unless item.nil?
			if !nested_ref.nil?
				nested_id = nested_ref.attr('target').sub('#', '')
				nested_item = backmatter_item(nested_id)
				references[nested_id] = nested_item unless nested_item.nil?
			end
		end

		return references
	end


	def exact_matches(term)
		query = "#{DICT_ENTRIES}[#{ORTH_EQUALS}]"
		params = { :term => term }

		tei_matches = xpathquery(query, params)

		return matches(tei_matches)
{'fraction_non_alphanumeric': 0.13739523495621056, 'fraction_numerical': 0.006968641114982578, 'mean_word_length': 4.3420523138833, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '778543', 'n_tokens_mistral': 4613, 'n_tokens_neox': 4123, 'n_words': 759}