<commit_before>/*
 * Copyright 2014 hbz NRW (http://www.hbz-nrw.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package actions;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import play.mvc.Results.Chunks;
import helper.Globals;
import models.Node;

/**
 * @author <NAME>
 *
 */
public class Index {
    Chunks.Out<String> messageOut;

    /**
     * @param pid
     *            The namespaced pid to remove from index
     * @param index
     *            the elasticsearch index
     * @param type
     *            the type of the resource
     * @return A short message
     */
    public String remove(String pid, String index, String type) {

	Globals.search.delete(pid, index, type);
	return pid + " of type " + type + " removed from index " + index + "!";
    }

    /**
     * @param p
     *            The pid with namespace that must be indexed
     * @param index
     *            the name of the index. Convention is to use the namespace of
     *            the pid.
     * @param type
     *            the type of the resource
     * @return a short message.
     */
    public String index(String p, String index, String type) {
	String jsonCompactStr = new Transform().oaiore(p,
		"application/json+compact");
	Globals.search.index(index, type, p, jsonCompactStr);
	return p + " indexed in " + index + "!";
    }

    /**
     * @param p
     *            The pid with namespace that must be indexed
     * @param index
     *            the name of the index. Convention is to use the namespace of
     *            the pid.
     * @param type
     *            the type of the resource
     * @return a short message.
     */
    public String publicIndex(String p, String index, String type) {
	Globals.search.index(index, type, p, new Read().readNode(p).toString());
	return p + " indexed!";
    }

    protected String index(Node n) {
	String namespace = n.getNamespace();
	String pid = n.getPid();
	String msg = "";
	if ("public".equals(n.getPublishScheme())) {
	    msg = index(pid, "public_" + namespace, n.getContentType());
	} else {
	    msg = remove(pid, "public_" + namespace, n.getContentType());
	}
	return msg + "\n" + index(pid, namespace, n.getContentType());
    }

    /**
     * @param indexName
     */
    public void indexAll(String indexName) {
	Read read = new Read();
	String indexNameWithDatestamp = indexName + "-" + getCurrentDate();
	int until = 0;
	int stepSize = 10;
	int from = 0 - stepSize;
	List<String> nodes = new ArrayList<String>();
	do {
	    until += stepSize;
	    from += stepSize;
	    nodes = read.listRepoNamespace(indexName, from, until);
	    if (nodes.isEmpty())
		break;
	    messageOut.write(Globals.search.indexAll(read.getNodes(nodes),
		    indexNameWithDatestamp).toString());

	} while (nodes.size() == stepSize);
	messageOut.write("\nSuccessfuly Finished\n");
	messageOut.close();

    }

    private String getCurrentDate() {
	DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
	Date date = new Date();
	return dateFormat.format(date);
    }

    /**
     * @param out
     *            messages for chunked responses
     */
    public void setMessageQueue(Chunks.Out<String> out) {
	messageOut = out;<commit_msg>first attempt to increase performance of index all<commit_after>/*
 * Copyright 2014 hbz NRW (http://www.hbz-nrw.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package actions;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import play.mvc.Results.Chunks;
import helper.Globals;
import models.Node;

/**
 * @author <NAME>
 *
 */
public class Index {
    Chunks.Out<String> messageOut;

    /**
     * @param pid
     *            The namespaced pid to remove from index
     * @param index
     *            the elasticsearch index
     * @param type
     *            the type of the resource
     * @return A short message
     */
    public String remove(String pid, String index, String type) {

	Globals.search.delete(pid, index, type);
	return pid + " of type " + type + " removed from index " + index + "!";
    }

    /**
     * @param p
     *            The pid with namespace that must be indexed
     * @param index
     *            the name of the index. Convention is to use the namespace of
     *            the pid.
     * @param type
     *            the type of the resource
     * @return a short message.
     */
    public String index(String p, String index, String type) {
	String jsonCompactStr = new Transform().oaiore(p,
		"application/json+compact");
	Globals.search.index(index, type, p, jsonCompactStr);
	return p + " indexed in " + index + "!";
    }

    /**
     * @param p
     *            The pid with namespace that must be indexed
     * @param index
     *            the name of the index. Convention is to use the namespace of
     *            the pid.
     * @param type
     *            the type of the resource
     * @return a short message.
     */
    public String publicIndex(String p, String index, String type) {
	Globals.search.index(index, type, p, new Read().readNode(p).toString());
	return p + " indexed!";
    }

    protected String index(Node n) {
	String namespace = n.getNamespace();
	String pid = n.getPid();
	String msg = "";
	if ("public".equals(n.getPublishScheme())) {
	    msg = index(pid, "public_" + namespace, n.getContentType());
	} else {
	    msg = remove(pid, "public_" + namespace, n.getContentType());
	}
	return msg + "\n" + index(pid, namespace, n.getContentType());
    }

    /**
     * @param indexName
     */
    public void indexAll(String indexName) {
	Read read = new Read();
	String indexNameWithDatestamp = indexName + "-" + getCurrentDate();
	int until = 0;
	int stepSize = 10;
	int from = 0 - stepSize;
	List<String> nodes = read.listRepoNamespace(indexName);
	do {
	    until += stepSize;
	    from += stepSize;
	    if (nodes.isEmpty())
		break;
	    messageOut.write(Globals.search.indexAll(
		    read.getNodes(nodes.subList(from, until)),
		    indexNameWithDatestamp).toString());
	} while (until < nodes.size());
	messageOut.write("\nSuccessfuly Finished\n");
	messageOut.close();
    }

    private String getCurrentDate() {
	DateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
	Date date = new Date();
	return dateFormat.format(date);
    }

    /**
     * @param out
     *            messages for chunked responses
     */
    public void setMessageQueue(Chunks.Out<String> out) {
	messageOut = out;
{'fraction_non_alphanumeric': 0.11277705345501955, 'fraction_numerical': 0.0031290743155149934, 'mean_word_length': 2.6252362948960304, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733913', 'n_tokens_mistral': 2321, 'n_tokens_neox': 2234, 'n_words': 743}