<commit_before>package com.bagri.xquery.saxon;

import static com.bagri.core.Constants.bg_schema;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.bagri.core.api.SchemaRepository;
import com.bagri.core.api.BagriException;
import com.bagri.core.query.ExpressionContainer;
import com.bagri.core.server.api.DocumentManagement;
import com.bagri.core.server.api.QueryManagement;

import net.sf.saxon.expr.XPathContext;
import net.sf.saxon.lib.Resource;
import net.sf.saxon.lib.ResourceCollection;
import net.sf.saxon.om.SpaceStrippingRule;
import net.sf.saxon.trans.XPathException;

public class ResourceCollectionImpl implements ResourceCollection {
	
    private static final Logger logger = LoggerFactory.getLogger(ResourceCollectionImpl.class);
    
    private static final Map<Long, Resource> resourceCache = new ConcurrentHashMap<Long, Resource>();
	
	private String uri;
    private SchemaRepository repo;
	private ExpressionContainer query;
	private Collection<Long> docIds = null;
	private Iterator<Long> iter = null;
	
	
	public ResourceCollectionImpl(String uri, Collection<Long> docIds) {
		this.uri = uri;
		this.docIds = new ArrayList<>(docIds);
		this.iter = docIds.iterator();
	}

	public ResourceCollectionImpl(String uri, SchemaRepository repo, ExpressionContainer query) {
		this.uri = uri;
		this.repo = repo;
		this.query = query;
	}
	
	private void loadData() { 
		try {
			docIds = ((QueryManagement) repo.getQueryManagement()).getDocumentIds(query);
		} catch (BagriException ex) {
			logger.error("loadData.error;", ex);
			docIds = Collections.emptyList();
		}
		logger.trace("loadData; got {} document ids", docIds.size());
		this.iter = docIds.iterator();
	}
	
	private boolean hasNext() {
		if (docIds == null) {
			loadData();
		}
		boolean result = iter.hasNext();
		logger.trace("hasNext; returning: {}", result);
		return result;
	}
	
	private Long next() { 
		if (docIds == null) {
			loadData();
		}
		Long currentId = iter.next();
		logger.trace("next; returning: {}", currentId);
		return currentId;
	}
	
	@Override
	public String getCollectionURI() {
		return uri;
	}

	@Override
	public Iterator<String> getResourceURIs(XPathContext context) throws XPathException {
		logger.trace("getResourceURIs.enter;");
		return new UriIterator();
	}

	@Override
	public Iterator<? extends Resource> getResources(XPathContext context) throws XPathException {
		logger.trace("getResources.enter;");
		return new ResourceIterator();
	}

	@Override
	public boolean isStable(XPathContext context) {
		return false; // true;
	}

	@Override
	public boolean stripWhitespace(SpaceStrippingRule rules) {
		logger.trace("stripWhitespace.enter; rules: {}", rules);
		return false;
	}

	@Override
	public String toString() {
		return "ResourceCollectionImpl [query=" + query	+ ", docIds=" + docIds  + "]";
	}
	
	public class UriIterator implements Iterator<String> {
		
		@Override
		public boolean hasNext() {
			return ResourceCollectionImpl.this.hasNext();
		}

		@Override
		public String next() {
			Long next = ResourceCollectionImpl.this.next();
			if (next != null) {
				return bg_schema + ":/" + next;
			}
			return null;
		}

		@Override
		public void remove() {
			throw new UnsupportedOperationException("method remove is not supported");
		}
		
	}

	public class ResourceIterator implements Iterator<Resource> {

		@Override
		public boolean hasNext() {
			return ResourceCollectionImpl.this.hasNext();
		}

		@Override
		public Resource next() {
			Long docKey = ResourceCollectionImpl.this.next();
			if (docKey != null) {
				Resource res = resourceCache.get(docKey);
				if (res == null) {
					try {
						DocumentManagement docMgr = (DocumentManagement) repo.getDocumentManagement(); 
						String type = docMgr.getDocumentContentType(docKey);
						
						// we can check isDocVisible here, thus don't do this at loadData phase.. 
						if ("MAP".equals(type)) {
							logger.trace("ResourceIterator.next; returning new MapResource for docKey {}", docKey);
					        res = new MapResourceImpl(docMgr, docKey);
						} else if ("JSON".equals(type)) {
					        res = new JsonResourceImpl(docMgr, docKey);
						} else {
							res = new XmlResourceImpl(docMgr, docKey);
						}
						resourceCache.putIfAbsent(docKey, res);
					} catch (BagriException ex) {
						logger.error("next.error", ex);
					}
				}
				return res;
			}
			return null;<commit_msg>trying weak HM<commit_after>package com.bagri.xquery.saxon;

import static com.bagri.core.Constants.bg_schema;

import java.lang.ref.Reference;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.bagri.core.api.SchemaRepository;
import com.bagri.core.api.BagriException;
import com.bagri.core.query.ExpressionContainer;
import com.bagri.core.server.api.DocumentManagement;
import com.bagri.core.server.api.QueryManagement;

import net.sf.saxon.expr.XPathContext;
import net.sf.saxon.lib.Resource;
import net.sf.saxon.lib.ResourceCollection;
import net.sf.saxon.om.SpaceStrippingRule;
import net.sf.saxon.trans.XPathException;

public class ResourceCollectionImpl implements ResourceCollection {
	
    private static final Logger logger = LoggerFactory.getLogger(ResourceCollectionImpl.class);
    
    //private static final Map<Long, Resource> resourceCache = new ConcurrentHashMap<Long, Resource>();
    private static final Map<Long, Reference<Resource>> resourceCache = Collections.synchronizedMap(new WeakHashMap<Long, Reference<Resource>>()); 
    
	private String uri;
    private SchemaRepository repo;
	private ExpressionContainer query;
	private Collection<Long> docIds = null;
	private Iterator<Long> iter = null;
	
	
	public ResourceCollectionImpl(String uri, Collection<Long> docIds) {
		this.uri = uri;
		this.docIds = new ArrayList<>(docIds);
		this.iter = docIds.iterator();
	}

	public ResourceCollectionImpl(String uri, SchemaRepository repo, ExpressionContainer query) {
		this.uri = uri;
		this.repo = repo;
		this.query = query;
	}
	
	private void loadData() { 
		try {
			docIds = ((QueryManagement) repo.getQueryManagement()).getDocumentIds(query);
		} catch (BagriException ex) {
			logger.error("loadData.error;", ex);
			docIds = Collections.emptyList();
		}
		logger.trace("loadData; got {} document ids", docIds.size());
		this.iter = docIds.iterator();
	}
	
	private boolean hasNext() {
		if (docIds == null) {
			loadData();
		}
		boolean result = iter.hasNext();
		logger.trace("hasNext; returning: {}", result);
		return result;
	}
	
	private Long next() { 
		if (docIds == null) {
			loadData();
		}
		Long currentId = iter.next();
		logger.trace("next; returning: {}", currentId);
		return currentId;
	}
	
	@Override
	public String getCollectionURI() {
		return uri;
	}

	@Override
	public Iterator<String> getResourceURIs(XPathContext context) throws XPathException {
		logger.trace("getResourceURIs.enter;");
		return new UriIterator();
	}

	@Override
	public Iterator<? extends Resource> getResources(XPathContext context) throws XPathException {
		logger.trace("getResources.enter;");
		return new ResourceIterator();
	}

	@Override
	public boolean isStable(XPathContext context) {
		return false; // true;
	}

	@Override
	public boolean stripWhitespace(SpaceStrippingRule rules) {
		logger.trace("stripWhitespace.enter; rules: {}", rules);
		return false;
	}

	@Override
	public String toString() {
		return "ResourceCollectionImpl [query=" + query	+ ", docIds=" + docIds  + "]";
	}
	
	public class UriIterator implements Iterator<String> {
		
		@Override
		public boolean hasNext() {
			return ResourceCollectionImpl.this.hasNext();
		}

		@Override
		public String next() {
			Long next = ResourceCollectionImpl.this.next();
			if (next != null) {
				return bg_schema + ":/" + next;
			}
			return null;
		}

		@Override
		public void remove() {
			throw new UnsupportedOperationException("method remove is not supported");
		}
		
	}

	public class ResourceIterator implements Iterator<Resource> {

		@Override
		public boolean hasNext() {
			return ResourceCollectionImpl.this.hasNext();
		}

		@Override
		public Resource next() {
			Long docKey = ResourceCollectionImpl.this.next();
			if (docKey != null) {
				Reference<Resource> ref = resourceCache.get(docKey);
				if (ref == null) {
					Resource res;
					try {
						DocumentManagement docMgr = (DocumentManagement) repo.getDocumentManagement(); 
						String type = docMgr.getDocumentContentType(docKey);
						
						// we can check isDocVisible here, thus don't do this at loadData phase.. 
						if ("MAP".equals(type)) {
							logger.trace("ResourceIterator.next; returning new MapResource for docKey {}", docKey);
					        res = new MapResourceImpl(docMgr, docKey);
						} else if ("JSON".equals(type)) {
					        res = new JsonResourceImpl(docMgr, docKey);
						} else {
							res = new XmlResourceImpl(docMgr, docKey);
						}
						ref = new WeakReference<Resource>(res);
						resourceCache.putIfAbsent(docKey, ref);
					} catch (BagriException ex) {
						logger.error("next.error", ex);
					}
				}
				return ref.get();
			}
			return null;
{'fraction_non_alphanumeric': 0.11658628643321452, 'fraction_numerical': 0.0004193751310547285, 'mean_word_length': 4.015247108307046, 'pattern_counts': {'":': 2, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1469455', 'n_tokens_mistral': 3276, 'n_tokens_neox': 2849, 'n_words': 685}