<commit_before># Author:: <NAME>
# Copyright:: (c) 2005-2006 <NAME>
# License:: LGPL

require 'sqlite3'
require 'active_rdf'
require 'federation/connection_pool'
require 'uuidtools'

$activerdflog.info "loading RDFLite adapter"

begin 
	require 'ferret'
	@@have_ferret = true
rescue LoadError
	$activerdflog.info "Keyword search is disabled since we could not load Ferret. To 
	enable, please do \"gem install ferret\""
	@@have_ferret = false
end

# RDFLite is a lightweight RDF database on top of sqlite3. It can act as adapter 
# in ActiveRDF. It supports on-disk and in-memory usage, and allows keyword 
# search if ferret is installed.
class RDFLite < ActiveRdfAdapter
	ConnectionPool.register_adapter(:rdflite,self)
	bool_accessor :keyword_search, :reasoning

	# instantiates RDFLite database
	# available parameters:
	# * :location => filepath (defaults to memory)
	# * :keyword => true/false (defaults to true)
	# * :pidx, :oidx, etc. => true/false (enable/disable these indices)
	def initialize(params = {})
		$activerdflog.info "initialised rdflite with params #{params.to_s}"

		@reads = true
		@writes = true

		# if no file-location given, we use in-memory store
		file = params[:location] || ':memory:'
		@db = SQLite3::Database.new(file) 

		# enable keyword search by default, but only if ferret is found
		@keyword_search = params[:keyword].nil? ? true : params[:keyword]
		@keyword_search &= @@have_ferret

		@reasoning = params[:reasoning] || false

		if keyword_search?
			# we initialise the ferret index, either as a file or in memory
			infos = Ferret::Index::FieldInfos.new

			# we setup the fields not to store object's contents
			infos.add_field(:subject, :store => :yes, :index => :no, :term_vector => :no)
			infos.add_field(:object, :store => :no) #, :index => :omit_norms)

			@ferret = if params[:location]
									Ferret::I.new(:path => params[:location] + '.ferret', :field_infos => infos)
								else
									Ferret::I.new(:field_infos => infos)
								end
		end

		# turn off filesystem synchronisation for speed
		@db.synchronous = 'off'

		# create triples table. ignores duplicated triples
		@db.execute('create table if not exists triple(s,p,o,c, unique(s,p,o,c) on conflict ignore)')

		create_indices(params)
		@db
	end

	# returns the number of triples in the datastore (incl. possible duplicates)
	def size
		@db.execute('select count(*) from triple')[0][0].to_i
	end

	# returns all triples in the datastore
	def dump
		@db.execute('select s,p,o,c from triple').collect do |s,p,o,c|
			[s,p,o,c].join(' ')
		end
	end

	# deletes all triples from datastore
	def clear
		@db.execute('delete from triple')
	end

	# deletes triple(s,p,o,c) from datastore
	# symbol parameters match anything: delete(:s,:p,:o) will delete all triples
	# you can specify a context to limit deletion to that context: 
	# delete(:s,:p,:o, 'http://context') will delete all triples with that context
	def delete(s, p, o, c=nil)
		# convert non-nil input to internal format
		quad = [s,p,o,c].collect {|r| r.nil? ? nil : internalise(r) }

		# construct where clause for deletion (for all non-nil input)
		where_clauses = []
		conditions = []		
		quad.each_with_index do |r,i|
			unless r.nil?
				conditions << r
				where_clauses << "#{SPOC[i]} = ?"
			end
		end

		# construct delete string
		ds = 'delete from triple'
		ds << " where #{where_clauses.join(' and ')}" unless where_clauses.empty?

		# execute delete string with possible deletion conditions (for each 
		# non-empty where clause)
		$activerdflog.debug("deleting #{[s,p,o,c].join(' ')}")
		@db.execute(ds, *conditions)

		# delete literal from ferret index
		@ferret.search_each("subject:\"#{s}\", object:\"#{o}\"") do |idx, score|
			@ferret.delete(idx)
		end if keyword_search?

		@db
	end

	# adds triple(s,p,o) to datastore
	# s,p must be resources, o can be primitive data or resource
	def add(s,p,o,c=nil)
		# check illegal input
		raise(ActiveRdfError, "adding non-resource #{s} while adding (#{s},#{p},#{o},#{c})") unless s.respond_to?(:uri)
		raise(ActiveRdfError, "adding non-resource #{p} while adding (#{s},#{p},#{o},#{c})") unless p.respond_to?(:uri)

		# get internal representation (array)
		quad = [s,p,o,c].collect {|r| internalise(r) }

		# insert the triple into the datastore
		@db.execute('insert into triple values (?,?,?,?)', *quad)

		# if keyword-search available, insert the object into keyword search
		@ferret << {:subject => s, :object => o} if keyword_search?
	end

	# flushes openstanding changes to underlying sqlite3
	def flush
		# since we always write changes into sqlite3 immediately, we don't do 
		# anything here
		true
	end

	# loads triples from file in ntriples format
	def load(file)
		ntriples = File.readlines(file)
		$activerdflog.debug "read #{ntriples.size} triples from file #{file}"

		# use filename as context
		context = internalise(RDFS::Resource.new("file:#{file}"))

		add_ntriples(ntriples, context)
	end

	# adds ntriples from given context into datastore
  # TODO: use NTriplesParser
	def add_ntriples(ntriples, context)

		# need unique identifier for this batch of triples (to detect occurence of 
		# same bnodes _:#1
		uuid = UUID.random_create.to_s

		# add each triple to db
		@db.transaction
		insert = @db.prepare('insert into triple values (?,?,?,?);')

		ntriples.each do |triple|
			nodes = triple.scan(Node)

			# handle bnodes if necessary (bnodes need to have uri generated)
			subject = case nodes[0]
								when BNode
									"<http://www.activerdf.org/bnode/#{uuid}/#$1>"
								else
									nodes[0]
								end

			predicate = nodes[1]

			# handle bnodes and literals if necessary (literals need unicode fixing)
			object = case nodes[2]
							 when BNode
								 "<http://www.activerdf.org/bnode/#{uuid}/#$1>"
							 when Literal
								 fix_unicode(nodes[2])
							 else
								 nodes[2]
							 end

			# insert triple into database
			insert.execute(subject, predicate, object, context)

			# if keyword-search available, insert the object into keyword search
			@ferret << {:subject => subject, :object => object} if keyword_search?
		end

		@db.commit
		@db
	end

	# executes ActiveRDF query on datastore
	def query(query)
		# construct query clauses
		sql, conditions = translate(query)

		# executing query, passing all where-clause values as parameters (so that 
		# sqlite will encode quotes correctly)
		results = @db.execute(sql, *conditions)

		# if ASK query, we check whether we received a positive result count
		if query.ask?
			return [[results[0][0].to_i > 0]]
		elsif query.count?
			return [[results[0][0].to_i]]
		else
			# otherwise we convert results to ActiveRDF nodes and return them
			return wrap(query, results)
		end
	end

	# translates ActiveRDF query into internal sqlite query string
	def translate(query)
		where, conditions = construct_where(query)
		[construct_select(query) + construct_join(query) + where + construct_sort(query) + construct_limit(query), conditions]
	end

	private
	# constants for extracting resources/literals from sql results
	BNode = /_:(\S*)/
	Resource = /<([^>]*)>/
	Literal = /"([^"]*)"/
	Node = Regexp.union(/_:\S*/,/<[^>]*>/,/"[^"]*"/)
	SPOC = ['s','p','o','c']

	# construct select clause
	def construct_select(query)
		# ASK queries counts the results, and return true if results > 0
		return "select count(*)" if query.ask?

		# add select terms for each selectclause in the query
		# the term names depend on the join conditions, e.g. t0.s or t1.p
		select = query.select_clauses.collect do |term|
			variable_name(query, term)
		end

		# add possible distinct and count functions to select clause
		select_clause = ''
		select_clause << 'distinct ' if query.distinct?
		select_clause << select.join(', ')
		select_clause = "count(#{select_clause})" if query.count?

		"select " + select_clause
	end

	# construct (optional) limit and offset clauses
	def construct_limit(query)
		clause = ""

		# if no limit given, use limit -1 (no limit)
		limit = query.limits.nil? ? -1 : query.limits

		# if no offset given, use offset 0
		offset = query.offsets.nil? ? 0 : query.offsets

		clause << " limit #{limit} offset #{offset}"
		clause
	end

	# sort query results on variable clause (optionally)
	def construct_sort(query)
		return "" if query.sort_clauses.empty?

		sort = query.sort_clauses.collect do |term|
			variable_name(query, term)
		end
		" order by (#{sort.join(',')})"
	end

	# construct join clause
	# TODO: joins don't work this way, they have to be linear (in one direction 
	# only, and we should only alias tables we didnt alias yet)
	# we should only look for one join clause in each where-clause: when we find 
	# one, we skip the rest of the variables in this clause.
	def construct_join(query)
		join_stmt = ''

		# no join necessary if only one where clause given
		return ' from triple as t0 ' if query.where_clauses.size == 1

		where_clauses = query.where_clauses.flatten
		considering = where_clauses.uniq.select{|w| w.is_a?(Symbol)}

		# constructing hash with indices for all terms
		# e.g. {?s => [1,3,5], ?p => [2], ... }
		term_occurrences = Hash.new()
		where_clauses.each_with_index do |term, index|
			ary = (term_occurrences[term] ||= [])
			ary << index 
		end

		aliases = {}

		where_clauses.each_with_index do |term, index|
			# if the term has been joined with his buddy already, we can skip it
			next unless considering.include?(term)

			# we find all (other) occurrences of this term
			indices = term_occurrences[term]

			# if the term doesnt have a join-buddy, we can skip it
			next if indices.size == 1

			# construct t0,t1,... as aliases for term
			# and construct join condition, e.g. t0.s
			termalias = "t#{index / 4}"
			termjoin = "#{termalias}.#{SPOC[index % 4]}"

			join = if join_stmt.include?(termalias)
							 ""
						 else
							 "triple as #{termalias}"
						 end

			indices.each do |i|
				# skip the current term itself
				next if i==index

				# construct t0,t1, etc. as aliases for buddy,
				# and construct join condition, e.g. t0.s = t1.p
				buddyalias = "t#{i/4}"
				buddyjoin = "#{buddyalias}.#{SPOC[i%4]}"

				# TODO: fix reuse of same table names as aliases, e.g.
				# "from triple as t1 join triple as t2 on ... join t1 on ..."
				# is not allowed as such by sqlite
				# but on the other hand, restating the aliases gives ambiguity:
				# "from triple as t1 join triple as t2 on ... join triple as t1 ..."
				# is ambiguous
				if join_stmt.include?(buddyalias)
					join << "and #{termjoin} = #{buddyjoin}"
				else
					join << " join triple as #{buddyalias} on #{termjoin} = #{buddyjoin} "
				end
			end
			join_stmt << join
			
			# remove term from 'todo' list of still-considered terms
			considering.delete(term)
		end

		if join_stmt == ''
			return " from triple as t0 "
		else
			return " from #{join_stmt} "
		end
	end

	# construct where clause
	def construct_where(query)
		# collecting where clauses, these will be added to the sql string later
		where = []

		# collecting all the right-hand sides of where clauses (e.g. where name = 
		# 'abc'), to add to query string later using ?-notation, because then 
		# sqlite will automatically encode quoted literals correctly
		right_hand_sides = []

		# convert each where clause to SQL:
		# add where clause for each subclause, except if it's a variable
		query.where_clauses.each_with_index do |clause,level|
			raise ActiveRdfError, "where clause #{clause} is not a triple" unless clause.is_a?(Array)
			clause.each_with_index do |subclause, i|
				# dont add where clause for variables
				unless subclause.is_a?(Symbol) || subclause.nil?
					conditions = compute_where_condition(i, subclause, query.reasoning? && reasoning?)
					if conditions.size == 1
						where << "t#{level}.#{SPOC[i]} = ?"
						right_hand_sides << conditions.first
					else
						conditions = conditions.collect {|c| "'#{c}'"}
						where << "t#{level}.#{SPOC[i]} in (#{conditions.join(',')})"
					end
				end
			end
		end

		# if keyword clause given, convert it using keyword index
		if query.keyword?
			subjects = []
			select_subject = query.keywords.collect {|subj,key| subj}.uniq
			raise ActiveRdfError, "cannot do keyword search over multiple subjects" if select_subject.size > 1

			keywords = query.keywords.collect {|subj,key| key}
			@ferret.search_each("object:#{keywords}") do |idx,score|
				subjects << @ferret[idx][:subject]
			end
			subjects.uniq! if query.distinct?
			where << "#{variable_name(query,select_subject.first)} in (#{subjects.collect {'?'}.join(',')})"
			right_hand_sides += subjects
		end

		if where.empty?
			['',[]]
		else
			["where " + where.join(' and '), right_hand_sides]
		end
	end

	def compute_where_condition(index, subclause, reasoning)
		conditions = [subclause]

		# expand conditions with rdfs rules if reasoning enabled
		if reasoning
			case index
			when 0: ;
				# no rule for subjects
			when 1:
				# expand properties to include all subproperties
				conditions = subproperties(subclause) if subclause.respond_to?(:uri)
			when 2:
				# no rule for objects
			when 3:
				# no rule for contexts
			end
		end

		# convert conditions into internal format
		conditions.collect { |c| c.respond_to?(:uri) ? "<#{c.uri}>" : c.to_s }
	end

	def subproperties(resource)
		subproperty = Namespace.lookup(:rdfs,:subPropertyOf)
		children_query = Query.new.distinct(:sub).where(:sub, subproperty, resource)
		children_query.reasoning = false
		children = children_query.execute

		if children.empty?
			[resource]
		else
			[resource] + children.collect{|c| subproperties(c)}.flatten.compact
		end
	end

	# returns sql variable name for a queryterm
	def variable_name(query,term)
		# look up the first occurence of this term in the where clauses, and compute 
		# the level and s/p/o position of it
		index = query.where_clauses.flatten.index(term)

		if index.nil? 
			# term does not appear in where clause
			# but maybe it appears in a keyword clause
				
			# index would not be nil if we had:
			# select(:o).where(knud, knows, :o).where(:o, :keyword, 'eyal')
			#
			# the only possibility that index is nil is if we have:
			# select(:o).where(:o, :keyword, :eyal) (selecting subject)
			# or if we use a select clause that does not appear in any where clause

			# so we check if we find the term in the keyword clauses, otherwise we throw 
			# an error
			if query.keywords.keys.include?(term)
				return "t0.s"
			else
				raise ActiveRdfError,'unbound variable in select clause'
			end
		end

		termtable = "t#{index / 4}"
		termspo = SPOC[index % 4]
		return "#{termtable}.#{termspo}"
	end

	# wrap resources into ActiveRDF resources, literals into Strings
	def wrap(query, results)
		results.collect do |row|
			row.collect { |result| parse(result) }
		end
	end

	def parse(result)
		case result
		when Resource
			RDFS::Resource.new($1)
		when Literal
      # replace special characters to allow string interpolation for e.g. 'test\nbreak'
      $1.gsub('\n',"\n").gsub('\t',"\t")
		else
			# when we do a count(*) query we get a number, not a resource/literal
			result
		end
	end

	def create_indices(params)
		sidx = params[:sidx] || false
		pidx = params[:pidx] || false
		oidx = params[:oidx] || false
		spidx = params[:spidx] || true
		soidx = params[:soidx] || false
		poidx = params[:poidx] || true
		opidx = params[:opidx] || false

		# creating lookup indices
		@db.transaction do 
			@db.execute('create index if not exists sidx on triple(s)') if sidx
			@db.execute('create index if not exists pidx on triple(p)') if pidx
			@db.execute('create index if not exists oidx on triple(o)') if oidx
			@db.execute('create index if not exists spidx on triple(s,p)') if spidx
			@db.execute('create index if not exists soidx on triple(s,p)') if soidx
			@db.execute('create index if not exists poidx on triple(p,o)') if poidx
			@db.execute('create index if not exists opidx on triple(o,p)') if opidx
		end
	end

	# transform triple into internal format <uri> and "literal"
	# returns array [s,p,o] 
	def internalise(r)
		if r.respond_to?(:uri)
			"<#{r.uri}>"
		elsif r.is_a?(Symbol)
			nil
		else
			"\"#{r.to_s}\""
		end
	end

	# fixes unicode characters in literals (because we parse them wrongly somehow)
	def fix_unicode(str)
		tmp = str.gsub(/\\\u([0-9a-fA-F]{4,4})/u){ "U+#$1" }
    tmp.gsub(/U\+([0-9a-fA-F]{4,4})/u){["#$1".hex ].pack('U*')}
	end

	public :subproperties
end<commit_msg>rdflite now uses NTriplesParser instead of doing its own parsing<commit_after># Author:: <NAME>
# Copyright:: (c) 2005-2006 <NAME>
# License:: LGPL

require 'sqlite3'
require 'active_rdf'
require 'federation/connection_pool'
require 'uuidtools'
require 'queryengine/ntriples_parser'

$activerdflog.info "loading RDFLite adapter"

begin 
	require 'ferret'
	@@have_ferret = true
rescue LoadError
	$activerdflog.info "Keyword search is disabled since we could not load Ferret. To 
	enable, please do \"gem install ferret\""
	@@have_ferret = false
end

# RDFLite is a lightweight RDF database on top of sqlite3. It can act as adapter 
# in ActiveRDF. It supports on-disk and in-memory usage, and allows keyword 
# search if ferret is installed.
class RDFLite < ActiveRdfAdapter
	ConnectionPool.register_adapter(:rdflite,self)
	bool_accessor :keyword_search, :reasoning

	# instantiates RDFLite database
	# available parameters:
	# * :location => filepath (defaults to memory)
	# * :keyword => true/false (defaults to true)
	# * :pidx, :oidx, etc. => true/false (enable/disable these indices)
	def initialize(params = {})
		$activerdflog.info "initialised rdflite with params #{params.to_s}"

		@reads = true
		@writes = true

		# if no file-location given, we use in-memory store
		file = params[:location] || ':memory:'
		@db = SQLite3::Database.new(file) 

		# enable keyword search by default, but only if ferret is found
		@keyword_search = params[:keyword].nil? ? true : params[:keyword]
		@keyword_search &= @@have_ferret

		@reasoning = params[:reasoning] || false

		if keyword_search?
			# we initialise the ferret index, either as a file or in memory
			infos = Ferret::Index::FieldInfos.new

			# we setup the fields not to store object's contents
			infos.add_field(:subject, :store => :yes, :index => :no, :term_vector => :no)
			infos.add_field(:object, :store => :no) #, :index => :omit_norms)

			@ferret = if params[:location]
									Ferret::I.new(:path => params[:location] + '.ferret', :field_infos => infos)
								else
									Ferret::I.new(:field_infos => infos)
								end
		end

		# turn off filesystem synchronisation for speed
		@db.synchronous = 'off'

		# create triples table. ignores duplicated triples
		@db.execute('create table if not exists triple(s,p,o,c, unique(s,p,o,c) on conflict ignore)')

		create_indices(params)
		@db
	end

	# returns the number of triples in the datastore (incl. possible duplicates)
	def size
		@db.execute('select count(*) from triple')[0][0].to_i
	end

	# returns all triples in the datastore
	def dump
		@db.execute('select s,p,o,c from triple').collect do |s,p,o,c|
			[s,p,o,c].join(' ')
		end
	end

	# deletes all triples from datastore
	def clear
		@db.execute('delete from triple')
	end

	# deletes triple(s,p,o,c) from datastore
	# symbol parameters match anything: delete(:s,:p,:o) will delete all triples
	# you can specify a context to limit deletion to that context: 
	# delete(:s,:p,:o, 'http://context') will delete all triples with that context
	def delete(s, p, o, c=nil)
		# convert non-nil input to internal format
		quad = [s,p,o,c].collect {|r| r.nil? ? nil : internalise(r) }

		# construct where clause for deletion (for all non-nil input)
		where_clauses = []
		conditions = []		
		quad.each_with_index do |r,i|
			unless r.nil?
				conditions << r
				where_clauses << "#{SPOC[i]} = ?"
			end
		end

		# construct delete string
		ds = 'delete from triple'
		ds << " where #{where_clauses.join(' and ')}" unless where_clauses.empty?

		# execute delete string with possible deletion conditions (for each 
		# non-empty where clause)
		$activerdflog.debug("deleting #{[s,p,o,c].join(' ')}")
		@db.execute(ds, *conditions)

		# delete literal from ferret index
		@ferret.search_each("subject:\"#{s}\", object:\"#{o}\"") do |idx, score|
			@ferret.delete(idx)
		end if keyword_search?

		@db
	end

	# adds triple(s,p,o) to datastore
	# s,p must be resources, o can be primitive data or resource
	def add(s,p,o,c=nil)
		# check illegal input
		raise(ActiveRdfError, "adding non-resource #{s} while adding (#{s},#{p},#{o},#{c})") unless s.respond_to?(:uri)
		raise(ActiveRdfError, "adding non-resource #{p} while adding (#{s},#{p},#{o},#{c})") unless p.respond_to?(:uri)

		# get internal representation (array)
		quad = [s,p,o,c].collect {|r| internalise(r) }

		# insert the triple into the datastore
		@db.execute('insert into triple values (?,?,?,?)', *quad)

		# if keyword-search available, insert the object into keyword search
		@ferret << {:subject => s, :object => o} if keyword_search?
	end

	# flushes openstanding changes to underlying sqlite3
	def flush
		# since we always write changes into sqlite3 immediately, we don't do 
		# anything here
		true
	end

	# loads triples from file in ntriples format
	def load(file)
		ntriples = File.readlines(file)
		$activerdflog.debug "read #{ntriples.size} triples from file #{file}"

		# use filename as context
		context = internalise(RDFS::Resource.new("file:#{file}"))

		add_ntriples(ntriples, context)
	end

	# adds ntriples from given context into datastore
	def add_ntriples(ntriples, context)

		# add each triple to db
		@db.transaction
		insert = @db.prepare('insert into triple values (?,?,?,?);')

    ntriples = NTriplesParser.parse(ntriples)
    ntriples.each do |s,p,o|
      # convert triples into internal db format
      subject, predicate, object = [s,p,o].collect {|r| internalise(r) }

			# insert triple into database
			insert.execute(subject, predicate, object, context)

			# if keyword-search available, insert the object into keyword search
			@ferret << {:subject => subject, :object => object} if keyword_search?
		end

		@db.commit
		@db
	end

	# executes ActiveRDF query on datastore
	def query(query)
		# construct query clauses
		sql, conditions = translate(query)

		# executing query, passing all where-clause values as parameters (so that 
		# sqlite will encode quotes correctly)
		results = @db.execute(sql, *conditions)

		# if ASK query, we check whether we received a positive result count
		if query.ask?
			return [[results[0][0].to_i > 0]]
		elsif query.count?
			return [[results[0][0].to_i]]
		else
			# otherwise we convert results to ActiveRDF nodes and return them
			return wrap(query, results)
		end
	end

	# translates ActiveRDF query into internal sqlite query string
	def translate(query)
		where, conditions = construct_where(query)
		[construct_select(query) + construct_join(query) + where + construct_sort(query) + construct_limit(query), conditions]
	end

	private
	# constants for extracting resources/literals from sql results
	SPOC = ['s','p','o','c']

	# construct select clause
	def construct_select(query)
		# ASK queries counts the results, and return true if results > 0
		return "select count(*)" if query.ask?

		# add select terms for each selectclause in the query
		# the term names depend on the join conditions, e.g. t0.s or t1.p
		select = query.select_clauses.collect do |term|
			variable_name(query, term)
		end

		# add possible distinct and count functions to select clause
		select_clause = ''
		select_clause << 'distinct ' if query.distinct?
		select_clause << select.join(', ')
		select_clause = "count(#{select_clause})" if query.count?

		"select " + select_clause
	end

	# construct (optional) limit and offset clauses
	def construct_limit(query)
		clause = ""

		# if no limit given, use limit -1 (no limit)
		limit = query.limits.nil? ? -1 : query.limits

		# if no offset given, use offset 0
		offset = query.offsets.nil? ? 0 : query.offsets

		clause << " limit #{limit} offset #{offset}"
		clause
	end

	# sort query results on variable clause (optionally)
	def construct_sort(query)
		return "" if query.sort_clauses.empty?

		sort = query.sort_clauses.collect do |term|
			variable_name(query, term)
		end
		" order by (#{sort.join(',')})"
	end

	# construct join clause
	# TODO: joins don't work this way, they have to be linear (in one direction 
	# only, and we should only alias tables we didnt alias yet)
	# we should only look for one join clause in each where-clause: when we find 
	# one, we skip the rest of the variables in this clause.
	def construct_join(query)
		join_stmt = ''

		# no join necessary if only one where clause given
		return ' from triple as t0 ' if query.where_clauses.size == 1

		where_clauses = query.where_clauses.flatten
		considering = where_clauses.uniq.select{|w| w.is_a?(Symbol)}

		# constructing hash with indices for all terms
		# e.g. {?s => [1,3,5], ?p => [2], ... }
		term_occurrences = Hash.new()
		where_clauses.each_with_index do |term, index|
			ary = (term_occurrences[term] ||= [])
			ary << index 
		end

		aliases = {}

		where_clauses.each_with_index do |term, index|
			# if the term has been joined with his buddy already, we can skip it
			next unless considering.include?(term)

			# we find all (other) occurrences of this term
			indices = term_occurrences[term]

			# if the term doesnt have a join-buddy, we can skip it
			next if indices.size == 1

			# construct t0,t1,... as aliases for term
			# and construct join condition, e.g. t0.s
			termalias = "t#{index / 4}"
			termjoin = "#{termalias}.#{SPOC[index % 4]}"

			join = if join_stmt.include?(termalias)
							 ""
						 else
							 "triple as #{termalias}"
						 end

			indices.each do |i|
				# skip the current term itself
				next if i==index

				# construct t0,t1, etc. as aliases for buddy,
				# and construct join condition, e.g. t0.s = t1.p
				buddyalias = "t#{i/4}"
				buddyjoin = "#{buddyalias}.#{SPOC[i%4]}"

				# TODO: fix reuse of same table names as aliases, e.g.
				# "from triple as t1 join triple as t2 on ... join t1 on ..."
				# is not allowed as such by sqlite
				# but on the other hand, restating the aliases gives ambiguity:
				# "from triple as t1 join triple as t2 on ... join triple as t1 ..."
				# is ambiguous
				if join_stmt.include?(buddyalias)
					join << "and #{termjoin} = #{buddyjoin}"
				else
					join << " join triple as #{buddyalias} on #{termjoin} = #{buddyjoin} "
				end
			end
			join_stmt << join
			
			# remove term from 'todo' list of still-considered terms
			considering.delete(term)
		end

		if join_stmt == ''
			return " from triple as t0 "
		else
			return " from #{join_stmt} "
		end
	end

	# construct where clause
	def construct_where(query)
		# collecting where clauses, these will be added to the sql string later
		where = []

		# collecting all the right-hand sides of where clauses (e.g. where name = 
		# 'abc'), to add to query string later using ?-notation, because then 
		# sqlite will automatically encode quoted literals correctly
		right_hand_sides = []

		# convert each where clause to SQL:
		# add where clause for each subclause, except if it's a variable
		query.where_clauses.each_with_index do |clause,level|
			raise ActiveRdfError, "where clause #{clause} is not a triple" unless clause.is_a?(Array)
			clause.each_with_index do |subclause, i|
				# dont add where clause for variables
				unless subclause.is_a?(Symbol) || subclause.nil?
					conditions = compute_where_condition(i, subclause, query.reasoning? && reasoning?)
					if conditions.size == 1
						where << "t#{level}.#{SPOC[i]} = ?"
						right_hand_sides << conditions.first
					else
						conditions = conditions.collect {|c| "'#{c}'"}
						where << "t#{level}.#{SPOC[i]} in (#{conditions.join(',')})"
					end
				end
			end
		end

		# if keyword clause given, convert it using keyword index
		if query.keyword?
			subjects = []
			select_subject = query.keywords.collect {|subj,key| subj}.uniq
			raise ActiveRdfError, "cannot do keyword search over multiple subjects" if select_subject.size > 1

			keywords = query.keywords.collect {|subj,key| key}
			@ferret.search_each("object:#{keywords}") do |idx,score|
				subjects << @ferret[idx][:subject]
			end
			subjects.uniq! if query.distinct?
			where << "#{variable_name(query,select_subject.first)} in (#{subjects.collect {'?'}.join(',')})"
			right_hand_sides += subjects
		end

		if where.empty?
			['',[]]
		else
			["where " + where.join(' and '), right_hand_sides]
		end
	end

	def compute_where_condition(index, subclause, reasoning)
		conditions = [subclause]

		# expand conditions with rdfs rules if reasoning enabled
		if reasoning
			case index
			when 0: ;
				# no rule for subjects
			when 1:
				# expand properties to include all subproperties
				conditions = subproperties(subclause) if subclause.respond_to?(:uri)
			when 2:
				# no rule for objects
			when 3:
				# no rule for contexts
			end
		end

		# convert conditions into internal format
		conditions.collect { |c| c.respond_to?(:uri) ? "<#{c.uri}>" : c.to_s }
	end

	def subproperties(resource)
		subproperty = Namespace.lookup(:rdfs,:subPropertyOf)
		children_query = Query.new.distinct(:sub).where(:sub, subproperty, resource)
		children_query.reasoning = false
		children = children_query.execute

		if children.empty?
			[resource]
		else
			[resource] + children.collect{|c| subproperties(c)}.flatten.compact
		end
	end

	# returns sql variable name for a queryterm
	def variable_name(query,term)
		# look up the first occurence of this term in the where clauses, and compute 
		# the level and s/p/o position of it
		index = query.where_clauses.flatten.index(term)

		if index.nil? 
			# term does not appear in where clause
			# but maybe it appears in a keyword clause
				
			# index would not be nil if we had:
			# select(:o).where(knud, knows, :o).where(:o, :keyword, 'eyal')
			#
			# the only possibility that index is nil is if we have:
			# select(:o).where(:o, :keyword, :eyal) (selecting subject)
			# or if we use a select clause that does not appear in any where clause

			# so we check if we find the term in the keyword clauses, otherwise we throw 
			# an error
			if query.keywords.keys.include?(term)
				return "t0.s"
			else
				raise ActiveRdfError,'unbound variable in select clause'
			end
		end

		termtable = "t#{index / 4}"
		termspo = SPOC[index % 4]
		return "#{termtable}.#{termspo}"
	end

	# wrap resources into ActiveRDF resources, literals into Strings
	def wrap(query, results)
		results.collect do |row|
			row.collect { |result| parse(result) }
		end
	end

	def parse(result)
		case result
		when Literal
      # replace special characters to allow string interpolation for e.g. 'test\nbreak'
      $1.gsub('\n',"\n").gsub('\t',"\t")
		when Resource
			RDFS::Resource.new($1)
		else
			# when we do a count(*) query we get a number, not a resource/literal
			result
		end
	end

	def create_indices(params)
		sidx = params[:sidx] || false
		pidx = params[:pidx] || false
		oidx = params[:oidx] || false
		spidx = params[:spidx] || true
		soidx = params[:soidx] || false
		poidx = params[:poidx] || true
		opidx = params[:opidx] || false

		# creating lookup indices
		@db.transaction do 
			@db.execute('create index if not exists sidx on triple(s)') if sidx
			@db.execute('create index if not exists pidx on triple(p)') if pidx
			@db.execute('create index if not exists oidx on triple(o)') if oidx
			@db.execute('create index if not exists spidx on triple(s,p)') if spidx
			@db.execute('create index if not exists soidx on triple(s,p)') if soidx
			@db.execute('create index if not exists poidx on triple(p,o)') if poidx
			@db.execute('create index if not exists opidx on triple(o,p)') if opidx
		end
	end

	# transform triple into internal format <uri> and "literal"
	# returns array [s,p,o] 
	def internalise(r)
		if r.respond_to?(:uri)
			"<#{r.uri}>"
		elsif r.is_a?(Symbol)
			nil
		else
			"\"#{r.to_s}\""
		end
	end

	# fixes unicode characters in literals (because we parse them wrongly somehow)
	def fix_unicode(str)
		tmp = str.gsub(/\\\u([0-9a-fA-F]{4,4})/u){ "U+#$1" }
    tmp.gsub(/U\+([0-9a-fA-F]{4,4})/u){["#$1".hex ].pack('U*')}
	end

  Resource = /<([^>]*)>/
  Literal = /"([^"]*)"/

	public :subproperties
end
{'fraction_non_alphanumeric': 0.12112223458163392, 'fraction_numerical': 0.004789272030651341, 'mean_word_length': 3.60187686620219, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '669836', 'n_tokens_mistral': 12164, 'n_tokens_neox': 10585, 'n_words': 3483}