<commit_before>/*
 * Copyright 2016 Searsia
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.searsia.index;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Random;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.json.JSONException;
import org.json.JSONObject;
import org.searsia.engine.Resource;

/**
 * Stores resources in a Lucene index
 *
 * @author <NAME>
 */
public class ResourceIndex {

    private final static Logger LOGGER = Logger.getLogger(ResourceIndex.class.getName());

    private final static Version version      = Version.LUCENE_4_10_4;
    private final static int MAX_SOURCE_CACHE = 10000; // TODO: breaks if we discover more than 10000 sources
    
	private Map<String,Resource> engines = new LinkedHashMap<String,Resource>();
	private Random random   = new Random();
	private Resource mother = null;
	private Resource me     = null;
	private Path meFile     = null;
	private Path indexDir   = null;
	private IndexWriter writer = null;
	private String lastFlushed = null;

	/**
	 * Creates index or reads resources from index (if it exist)
	 * @param path path where the Searsia index resides
	 * @param filename index file name
	 * @throws IOException
	 * @throws JSONException 
	 */
	public ResourceIndex(String path, String filename) throws IOException, JSONException {
		this.meFile   = Paths.get(path, filename + ".json");
		this.indexDir = Paths.get(path, filename + "_sources");
		if (meFile.toFile().exists()) {
    		this.me = readMyselfFile(meFile);
		}
		if (this.indexDir.toFile().exists()) {
			readResourceIndex();			
		} else {
			this.indexDir.toFile().mkdir();
		}
		initResourceIndex();
	}


	private void writeMyselfFile(Resource engine) throws IOException {
	    JSONObject me = engine.toJson();
	    if (this.mother != null) {
    	    me.put("motherTemplate", this.mother.getAPITemplate());
	    }
	    Files.write(this.meFile, engine.toJson().toString().getBytes());
	}


	private Resource readMyselfFile(Path meFile) throws IOException, JSONException {
		String content = new String(Files.readAllBytes(meFile));
		Resource me = null;
		try {
			JSONObject json = new JSONObject(content);
			me = new Resource(json);
		} catch (javax.xml.xpath.XPathExpressionException e) {
			throw new IOException(e);
		}
		return me;
	}


	private void readResourceIndex() throws IOException {
        IndexReader reader = null;
        Directory dir = FSDirectory.open(this.indexDir.toFile());
        try {
            reader = DirectoryReader.open(dir); 
        }
        catch (org.apache.lucene.index.IndexNotFoundException e) { 
            LOGGER.warn("No resources in index.");
            return;
        }
        try {
            IndexSearcher searcher = new IndexSearcher(reader);
            ScoreDoc[] hits = searcher.search(new MatchAllDocsQuery(), MAX_SOURCE_CACHE).scoreDocs;
            for (ScoreDoc hit: hits) {
                Document doc = searcher.doc(hit.doc);
                JSONObject json = new JSONObject(doc.get("json"));
                Resource engine = new Resource((JSONObject) json.get("resource"));
                if (json.has("health")) {
                    engine.updateHealth((JSONObject) json.get("health"));
                    String lastUpdated = engine.getLastUpdatedString();
                    if (this.lastFlushed == null || this.lastFlushed.compareTo(lastUpdated) < 0) {
                        this.lastFlushed = lastUpdated;
                    }
                }
                this.engines.put(engine.getId(), engine);
            }
        } catch (Exception e) {
        	throw new IOException(e.getMessage());
        }
        finally {
            reader.close(); <commit_msg>resource index more lenient on index error<commit_after>/*
 * Copyright 2016 Searsia
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.searsia.index;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Random;

import javax.xml.xpath.XPathExpressionException;

import org.apache.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.json.JSONException;
import org.json.JSONObject;
import org.searsia.engine.Resource;

/**
 * Stores resources in a Lucene index
 *
 * @author <NAME>
 */
public class ResourceIndex {

    private final static Logger LOGGER = Logger.getLogger(ResourceIndex.class.getName());

    private final static Version version      = Version.LUCENE_4_10_4;
    private final static int MAX_SOURCE_CACHE = 10000; // TODO: breaks if we discover more than 10000 sources
    
	private Map<String,Resource> engines = new LinkedHashMap<String,Resource>();
	private Random random   = new Random();
	private Resource mother = null;
	private Resource me     = null;
	private Path meFile     = null;
	private Path indexDir   = null;
	private IndexWriter writer = null;
	private String lastFlushed = null;

	/**
	 * Creates index or reads resources from index (if it exist)
	 * @param path path where the Searsia index resides
	 * @param filename index file name
	 * @throws IOException
	 */
	public ResourceIndex(String path, String filename) throws IOException {
		this.meFile   = Paths.get(path, filename + ".json");
		this.indexDir = Paths.get(path, filename + "_sources");
		if (meFile.toFile().exists()) {
			try {
        		this.me = readMyselfFile(meFile);
			} catch (IOException e) {
				LOGGER.warn("Myself not found: " + e.getMessage());
        		meFile.toFile().delete();
        	}
		}
		if (this.indexDir.toFile().exists()) {
 			readResourceIndex();
		} else {
			this.indexDir.toFile().mkdir();
		}
		initResourceIndex();
	}


	private void writeMyselfFile(Resource engine) throws IOException {
	    JSONObject me = engine.toJson();
	    if (this.mother != null) {
    	    me.put("motherTemplate", this.mother.getAPITemplate());
	    }
	    Files.write(this.meFile, engine.toJson().toString().getBytes());
	}


	private Resource readMyselfFile(Path meFile) throws IOException {
		String content = new String(Files.readAllBytes(meFile));
		Resource me = null;
		try {
			JSONObject json = new JSONObject(content);
			me = new Resource(json);
		} catch (javax.xml.xpath.XPathExpressionException e) {
			throw new IOException(e);
		} catch (JSONException e) {
			throw new IOException(e);
		}
		return me;
	}


	private void readResourceIndex() throws IOException {
        IndexReader reader = null;
        Directory dir = FSDirectory.open(this.indexDir.toFile());
        try {
            reader = DirectoryReader.open(dir); 
        }
        catch (org.apache.lucene.index.IndexNotFoundException e) { 
            LOGGER.warn("No resources in index.");
            return;
        }
        try {
            IndexSearcher searcher = new IndexSearcher(reader);
            ScoreDoc[] hits = searcher.search(new MatchAllDocsQuery(), MAX_SOURCE_CACHE).scoreDocs;
            for (ScoreDoc hit: hits) {
                Document doc = searcher.doc(hit.doc);
                try{
                    JSONObject json = new JSONObject(doc.get("json"));
                    Resource engine = new Resource((JSONObject) json.get("resource"));
                    if (json.has("health")) {
                        engine.updateHealth((JSONObject) json.get("health"));
                        String lastUpdated = engine.getLastUpdatedString();
                        if (this.lastFlushed == null || this.lastFlushed.compareTo(lastUpdated) < 0) {
                            this.lastFlushed = lastUpdated;
                        }
                    }
                    this.engines.put(engine.getId(), engine);
                } catch (XPathExpressionException | JSONException | ParseException e) { 
                  LOGGER.warn("Garbled index: " + e.getLocalizedMessage());	
                }
            }
        } catch (IOException e) {
        	throw new IOException(e);
        } finally {
            reader.close(); 
{'fraction_non_alphanumeric': 0.10545963602426504, 'fraction_numerical': 0.004479701353243117, 'mean_word_length': 3.0514177693761813, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5319493', 'n_tokens_mistral': 3183, 'n_tokens_neox': 2990, 'n_words': 796}