<commit_before>/**********************************************************************************
 * $URL$
 * $Id$
 ***********************************************************************************
 *
 * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **********************************************************************************/

package org.sakaiproject.search.component.adapter.contenthosting;

import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sakaiproject.component.api.ServerConfigurationService;
import org.sakaiproject.content.api.ContentHostingService;
import org.sakaiproject.content.api.ContentResource;
import org.sakaiproject.entity.api.EntityManager;
import org.sakaiproject.entity.api.EntityProducer;
import org.sakaiproject.entity.api.Reference;
import org.sakaiproject.entity.api.ResourceProperties;
import org.sakaiproject.event.api.Event;
import org.sakaiproject.exception.IdUnusedException;
import org.sakaiproject.exception.PermissionException;
import org.sakaiproject.exception.TypeException;
import org.sakaiproject.search.api.EntityContentProducer;
import org.sakaiproject.search.api.SearchIndexBuilder;
import org.sakaiproject.search.api.SearchService;
import org.sakaiproject.search.api.StoredDigestContentProducer;
import org.sakaiproject.search.model.SearchBuilderItem;
import org.sakaiproject.site.api.SiteService;

public class ContentHostingContentProducer implements EntityContentProducer, StoredDigestContentProducer
{

	private static Log log = LogFactory.getLog(ContentHostingContentProducer.class);

	/**
	 * resolved dep
	 */
	private SearchService searchService = null;

	/**
	 * resolved dep
	 */
	private ContentHostingService contentHostingService = null;

	/**
	 * resolved dep
	 */
	private SearchIndexBuilder searchIndexBuilder = null;

	/**
	 * resolved dep
	 */
	private EntityManager entityManager = null;

	/**
	 * resolved dep
	 */
	private SiteService siteService = null;

	/**
	 * runtime injected
	 */
	private ArrayList<ContentDigester> digesters = new ArrayList<ContentDigester>();

	/**
	 * config injected dep
	 */
	private ContentDigester defaultDigester;

	private int readerSizeLimit = 1024 * 1024 * 2; // (2M)

	private int digesterSizeLimit = 1024 * 1024 * 5; // (5M)

	/**
	 * A list of custom properties in the form indexkey.entitykey;indexkey.entitykey;indexkey.entitykey;
	 */
	private List<String> customProperties = null;

	private ServerConfigurationService serverConfigurationService;
	
	
	public ContentHostingContentProducer() {
		customProperties = new ArrayList<String>();
		customProperties.add("dc_created.http://purl.org/dc/terms/created");
		customProperties.add("Tdc_publisher.http://purl.org/dc/elements/1.1/publisher");
		customProperties.add("Tdc_audience.http://purl.org/dc/terms/audience");
		customProperties.add("Tdc_subject.http://purl.org/dc/elements/1.1/subject");
		customProperties.add("Tdc_creator.http://purl.org/dc/elements/1.1/creator");
		customProperties.add("Tdc_educationlevel.http://purl.org/dc/terms/educationLevel");
		customProperties.add("Tdc_alternative.http://purl.org/dc/elements/1.1/alternative");
		customProperties.add("dc_issued.http://purl.org/dc/terms/issued");
		customProperties.add("Tdc_abstract.http://purl.org/dc/terms/abstract");
		customProperties.add("Tdc_contributor.http://purl.org/dc/elements/1.1/contributor");
		customProperties.add("tag.tag");
		
		//LOM fields
		customProperties.add("Tlom_role.http://ltsc.ieee.org/xsd/lomv1.0/role");
		customProperties.add("Tlom_coverage.http://ltsc.ieee.org/xsd/lomv1.0/coverage");
		customProperties.add("Tlom_status.http://ltsc.ieee.org/xsd/lomv1.0/status");
		//customProperties.add("lom_duration.http://ltsc.ieee.org/xsd/lomv1.0/duration");
		customProperties.add("Tlom_engagement.http://ltsc.ieee.org/xsd/lomv1.0/engagement");
		customProperties.add("Tlom_learning_resource_type.http://ltsc.ieee.org/xsd/lomv1.0/learning_resource_type");
		customProperties.add("Tlom_interactivity_level.http://ltsc.ieee.org/xsd/lomv1.0/interactivity_level");
		customProperties.add("Tlom_context_level.http://ltsc.ieee.org/xsd/lomv1.0/context_level");
		customProperties.add("Tlom_difficulty.http://ltsc.ieee.org/xsd/lomv1.0/difficulty");
		//customProperties.add("lom_learning_time.http://ltsc.ieee.org/xsd/lomv1.0/learning_time");
		customProperties.add("Tlom_assumed_knowledge.http://ltsc.ieee.org/xsd/lomv1.0/assumed_knowledge");
		customProperties.add("Tlom_technical_requirements.http://ltsc.ieee.org/xsd/lomv1.0/technical_requirements");
		customProperties.add("Tlom_install_remarks.http://ltsc.ieee.org/xsd/lomv1.0/install_remarks");
		customProperties.add("Tlom_other_requirements.http://ltsc.ieee.org/xsd/lomv1.0/other_requirements");
		customProperties.add("Tlom_level.http://ltsc.ieee.org/xsd/lomv1.0/granularity_level");
		customProperties.add("Tlom_structure.http://ltsc.ieee.org/xsd/lomv1.0/structure");
		customProperties.add("Tlom_relation.http://ltsc.ieee.org/xsd/lomv1.0/relation");
		customProperties.add("Tlom_annotation_id.http://ltsc.ieee.org/xsd/lomv1.0/annotation_id");
		//customProperties.add("lom_annotation_date.http://ltsc.ieee.org/xsd/lomv1.0/annotation_date");
		customProperties.add("Tlom_annotation_description.http://ltsc.ieee.org/xsd/lomv1.0/annotation_description");

	}

	public void init()
	{
		try
		{

			if ("true".equals(serverConfigurationService.getString("search.enable",
					"false")))
			{

				searchService.registerFunction(ContentHostingService.EVENT_RESOURCE_ADD);
				searchService
						.registerFunction(ContentHostingService.EVENT_RESOURCE_WRITE);
				searchService
						.registerFunction(ContentHostingService.EVENT_RESOURCE_REMOVE);<commit_msg>SRCH-83 updates to the fields that are indexed to match SAK-22004<commit_after>/**********************************************************************************
 * $URL$
 * $Id$
 ***********************************************************************************
 *
 * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **********************************************************************************/

package org.sakaiproject.search.component.adapter.contenthosting;

import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sakaiproject.component.api.ServerConfigurationService;
import org.sakaiproject.content.api.ContentHostingService;
import org.sakaiproject.content.api.ContentResource;
import org.sakaiproject.entity.api.EntityManager;
import org.sakaiproject.entity.api.EntityProducer;
import org.sakaiproject.entity.api.Reference;
import org.sakaiproject.entity.api.ResourceProperties;
import org.sakaiproject.event.api.Event;
import org.sakaiproject.exception.IdUnusedException;
import org.sakaiproject.exception.PermissionException;
import org.sakaiproject.exception.TypeException;
import org.sakaiproject.search.api.EntityContentProducer;
import org.sakaiproject.search.api.SearchIndexBuilder;
import org.sakaiproject.search.api.SearchService;
import org.sakaiproject.search.api.StoredDigestContentProducer;
import org.sakaiproject.search.model.SearchBuilderItem;
import org.sakaiproject.site.api.SiteService;

public class ContentHostingContentProducer implements EntityContentProducer, StoredDigestContentProducer
{

	private static Log log = LogFactory.getLog(ContentHostingContentProducer.class);

	/**
	 * resolved dep
	 */
	private SearchService searchService = null;

	/**
	 * resolved dep
	 */
	private ContentHostingService contentHostingService = null;

	/**
	 * resolved dep
	 */
	private SearchIndexBuilder searchIndexBuilder = null;

	/**
	 * resolved dep
	 */
	private EntityManager entityManager = null;

	/**
	 * resolved dep
	 */
	private SiteService siteService = null;

	/**
	 * runtime injected
	 */
	private ArrayList<ContentDigester> digesters = new ArrayList<ContentDigester>();

	/**
	 * config injected dep
	 */
	private ContentDigester defaultDigester;

	private int readerSizeLimit = 1024 * 1024 * 2; // (2M)

	private int digesterSizeLimit = 1024 * 1024 * 5; // (5M)

	/**
	 * A list of custom properties in the form indexkey.entitykey;indexkey.entitykey;indexkey.entitykey;
	 */
	private List<String> customProperties = null;

	private ServerConfigurationService serverConfigurationService;
	
	
	public ContentHostingContentProducer() {
		customProperties = new ArrayList<String>();
		customProperties.add("dc_created.http://purl.org/dc/terms/created");
		customProperties.add("Tdc_publisher.http://purl.org/dc/elements/1.1/publisher");
		customProperties.add("Tdc_audience.http://purl.org/dc/terms/audience");
		customProperties.add("Tdc_subject.http://purl.org/dc/elements/1.1/subject");
		customProperties.add("Tdc_creator.http://purl.org/dc/elements/1.1/creator");
		customProperties.add("Tdc_educationlevel.http://purl.org/dc/terms/educationLevel");
		customProperties.add("Tdc_alternative.http://purl.org/dc/elements/1.1/alternative");
		customProperties.add("dc_issued.http://purl.org/dc/terms/issued");
		customProperties.add("Tdc_abstract.http://purl.org/dc/terms/abstract");
		customProperties.add("Tdc_contributor.http://purl.org/dc/elements/1.1/contributor");
		customProperties.add("tag.tag");
		
		//LOM fields
		customProperties.add("Tlom_role.http://ltsc.ieee.org/xsd/lomv1.0/role");
		customProperties.add("Tlom_coverage.http://ltsc.ieee.org/xsd/lomv1.0/coverage");
		customProperties.add("Tlom_status.http://ltsc.ieee.org/xsd/lomv1.0/status");
		//customProperties.add("lom_duration.http://ltsc.ieee.org/xsd/lomv1.0/duration");
		customProperties.add("Tlom_engagement.http://ltsc.ieee.org/xsd/lomv1.0/engagement");
		customProperties.add("Tlom_learning_resource_type.http://ltsc.ieee.org/xsd/lomv1.0/learning_resource_type");
		customProperties.add("Tlom_interactivity_level.http://ltsc.ieee.org/xsd/lomv1.0/interactivity_level");
		customProperties.add("Tlom_context_level.http://ltsc.ieee.org/xsd/lomv1.0/context_level");
		customProperties.add("Tlom_difficulty.http://ltsc.ieee.org/xsd/lomv1.0/difficulty");
		//customProperties.add("lom_learning_time.http://ltsc.ieee.org/xsd/lomv1.0/learning_time");
		customProperties.add("Tlom_assumed_knowledge.http://ltsc.ieee.org/xsd/lomv1.0/assumed_knowledge");
		customProperties.add("Tlom_technical_requirements.http://ltsc.ieee.org/xsd/lomv1.0/technical_requirements");
		customProperties.add("Tlom_install_remarks.http://ltsc.ieee.org/xsd/lomv1.0/install_remarks");
		customProperties.add("Tlom_other_requirements.http://ltsc.ieee.org/xsd/lomv1.0/other_requirements");
		customProperties.add("Tlom_level.http://ltsc.ieee.org/xsd/lomv1.0/granularity_level");
		customProperties.add("Tlom_structure.http://ltsc.ieee.org/xsd/lomv1.0/structure");
		customProperties.add("Tlom_relation.http://ltsc.ieee.org/xsd/lomv1.0/relation");
		customProperties.add("Tlom_reviewer.http://ltsc.ieee.org/xsd/lomv1.0/reviewer");
		//customProperties.add("lom_review_date.http://ltsc.ieee.org/xsd/lomv1.0/review_date");
		customProperties.add("Tlom_review_comments.http://ltsc.ieee.org/xsd/lomv1.0/review_comments");

	}

	public void init()
	{
		try
		{

			if ("true".equals(serverConfigurationService.getString("search.enable",
					"false")))
			{

				searchService.registerFunction(ContentHostingService.EVENT_RESOURCE_ADD);
				searchService
						.registerFunction(ContentHostingService.EVENT_RESOURCE_WRITE);
				searchService
						.registerFunction(ContentHostingService.EVENT_RESOURCE_REMOVE);
{'fraction_non_alphanumeric': 0.16256536450322978, 'fraction_numerical': 0.015610581359581667, 'mean_word_length': 5.46693187468921, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1226158', 'n_tokens_mistral': 4521, 'n_tokens_neox': 4099, 'n_words': 571}