<commit_before>package whelk.servlet

import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.map.ObjectMapper
import org.picocontainer.Characteristics
import org.picocontainer.PicoContainer
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.component.Storage
import whelk.converter.marc.MarcFrameConverter
import whelk.harvester.CullingOaiPmhHarvester
import whelk.harvester.HarvestResult
import whelk.harvester.LibrisOaiPmhHarvester
import whelk.harvester.OaiPmhHarvester
import whelk.harvester.OaiPmhHarvester.BrokenRecordException
import whelk.util.PropertyLoader

import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.util.concurrent.Executors
import java.util.concurrent.RejectedExecutionException
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit

/**
 * Created by markus on 15-09-03.
 */
@Log
class OaiPmhImporterServlet extends HttpServlet {

    PicoContainer pico
    int scheduleDelaySeconds = 5
    Properties props = new Properties()
    private Map<String,ScheduledJob> jobs = [:]

    static String SETTINGS_PFX = "harvester:"
    static String DEFAULT_HARVESTER = "whelk.harvester.OaiPmhHarvester"
    static int DEFAULT_INTERVAL = 3600
    static String DEFAULT_SYSTEM = "XL"

    static final ObjectMapper mapper = new ObjectMapper()


    public OaiPmhImporterServlet() {
        log.info("Starting oaipmhimporter.")

        props = PropertyLoader.loadProperties("secret", "mysql")

        pico = Whelk.getPreparedComponentsContainer(props)

        pico.addComponent(OaiPmhHarvester.class)
        pico.addComponent(CullingOaiPmhHarvester.class)
        pico.addComponent(LibrisOaiPmhHarvester.class)
        pico.addComponent(new MarcFrameConverter())

        pico.start()

        log.info("Started ...")
    }

    @Override
    void doGet(HttpServletRequest request, HttpServletResponse response) {
        def storage = pico.getComponent(PostgreSQLComponent)
        String html, json
        if (jobs) {
            List services = props.scheduledServices.split(",")
            def state = [:]
            StringBuilder table = new StringBuilder("<table cellspacing=\"10\"><tr><th>&nbsp;</th>")
            table.append("<form method=\"post\">")

            Set catSet = new TreeSet<String>()

            for (service in services) {
                state[service] = storage.loadSettings(SETTINGS_PFX+service)
                state[service]["harvesterClass"] = props.getProperty(service + ".harvesterClass", DEFAULT_HARVESTER)
                state[service]["serviceUrl"] = props.getProperty(service + ".serviceUrl")
                state[service]["interval"] = props.getProperty(service + ".interval", "" + DEFAULT_INTERVAL)
                catSet.add("harvesterClass")
                catSet.add("serviceUrl")
                catSet.add("interval")
                catSet.addAll(state[service].keySet())
                table.append("<th>$service</th>")

            }
            table.append("</tr>")
            List categories = catSet.toList()

            int i = 0
            for (cat in categories) {
                table.append("<tr><td align=\"right\"><b>$cat</b></td>")
                for (collection in services) {
                    table.append("<td>${state.get(collection).get(cat) != null ? state.get(collection).get(cat) : "&nbsp;"}</td>")
                }
                table.append("</tr>")
            }
            table.append("<tr><td><input type=\"submit\" name=\"action_all\" value=\"stop all\"></td>")
            for (collection in services) {
                table.append("<td><input type=\"submit\" name=\"action_${collection}\" value=\"${jobs[collection]?.active ? "stop" : "start"}\">")
                if (jobs[collection] && !jobs[collection].active) {
                    String lastImportDate = jobs[collection].getLastImportValue().format("yyyy-MM-dd'T'HH:mm")
                    table.append("&nbsp;<input type=\"submit\" name=\"reset_${collection}\" value=\"reload $collection from\"/>&nbsp;<input type=\"datetime-local\" name=\"datevalue\" value=\"${lastImportDate}\"/>")
                }
                table.append("</td>")
            }
            table.append("</tr>")

            table.append("</form></table>")

            html = """
                <html><head><title>OAIPMH Harvester control panel</title></head>
                <body>
                ${table.toString()}
                </form>
                """
            json = mapper.writeValueAsString(state)
        } else {

            html = """
                <html><head><title>OAIPMH Harvest Disabled</title></head>
                <body>

                HARVESTER DISABLED<br/>
                """
            json = mapper.writeValueAsString(["state":"disabled"])
        }
        PrintWriter out = response.getWriter();

        if (request.getPathInfo() == "/json") {
            response.setContentType("application/json");
            out.print(json);
        } else {
            response.setContentType("text/html");
            out.print(html);
        }

        out.flush();
    }

    void doPost(HttpServletRequest request, HttpServletResponse response) {
        log.debug("Received post request. Got this: ${request.getParameterMap()}")
        for (reqs in request.getParameterNames()) {
            if (reqs == "action_all") {
                for (job in jobs) {
                    job.value.disable()
                }
            } else if (reqs.startsWith("reset_")) {
                log.debug("Loading job for ${reqs.substring(6)}")
                log.debug("Got these jobs: $jobs")
                def job = jobs.get(reqs.substring(6))
                Date startDate = Date.parse("yyyy-MM-dd'T'HH:mm", request.getParameter("datevalue"))
                log.debug("Resetting harvester for ${job.collection} to $startDate")
                job.setStartDate(startDate)
                //job.enable()
            } else if (reqs.startsWith("action_")) {
                jobs.get(reqs.substring(7)).toggleActive()
            }
        }
        response.sendRedirect(request.getRequestURL().toString())
    }

    Map loadOaiPmhSettings() {
        Map settings = pico.getComponent(PostgreSQLComponent.class).loadSettings("oaipmh")

    }

    void init() {
        log.info("Initializing OAIPMH harvester.")
        Storage storage = pico.getComponent(PostgreSQLComponent.class)
        List services = props.scheduledServices.split(",")

        ScheduledExecutorService ses = Executors.newScheduledThreadPool(services.size())

        for (service in services) {

            log.info("Setting up schedule for $service")
            int scheduleIntervalSeconds = props.getProperty(service + ".interval", "" + DEFAULT_INTERVAL) as int
            String harvesterClass = props.getProperty(service + ".harvesterClass", DEFAULT_HARVESTER)
            String serviceUrl = props.getProperty(service + ".serviceUrl")
            String username = props.getProperty(service + ".username")
            String password = props.getProperty(service + ".password")
            String sourceSystem = props.getProperty(service + ".sourceSystem", DEFAULT_SYSTEM)
            def job = new ScheduledJob(pico.getComponent(Class.forName(harvesterClass)), "${SETTINGS_PFX}${service}",
                    serviceUrl, username, password, sourceSystem, storage)
            jobs[service] = job

            try {
                ses.scheduleWithFixedDelay(job, scheduleDelaySeconds, scheduleIntervalSeconds, TimeUnit.SECONDS)
            } catch (RejectedExecutionException ree) {
                log.error("execution failed", ree)
            }
        }
        log.info("scheduler started")
    }
}

@Log<commit_msg>Remove use of deleted Storage interface<commit_after>package whelk.servlet

import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.map.ObjectMapper
import org.picocontainer.Characteristics
import org.picocontainer.PicoContainer
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.converter.marc.MarcFrameConverter
import whelk.harvester.CullingOaiPmhHarvester
import whelk.harvester.HarvestResult
import whelk.harvester.LibrisOaiPmhHarvester
import whelk.harvester.OaiPmhHarvester
import whelk.harvester.OaiPmhHarvester.BrokenRecordException
import whelk.util.PropertyLoader

import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.util.concurrent.Executors
import java.util.concurrent.RejectedExecutionException
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit

/**
 * Created by markus on 15-09-03.
 */
@Log
class OaiPmhImporterServlet extends HttpServlet {

    PicoContainer pico
    int scheduleDelaySeconds = 5
    Properties props = new Properties()
    private Map<String,ScheduledJob> jobs = [:]

    static String SETTINGS_PFX = "harvester:"
    static String DEFAULT_HARVESTER = "whelk.harvester.OaiPmhHarvester"
    static int DEFAULT_INTERVAL = 3600
    static String DEFAULT_SYSTEM = "XL"

    static final ObjectMapper mapper = new ObjectMapper()


    public OaiPmhImporterServlet() {
        log.info("Starting oaipmhimporter.")

        props = PropertyLoader.loadProperties("secret", "mysql")

        pico = Whelk.getPreparedComponentsContainer(props)

        pico.addComponent(OaiPmhHarvester.class)
        pico.addComponent(CullingOaiPmhHarvester.class)
        pico.addComponent(LibrisOaiPmhHarvester.class)
        pico.addComponent(new MarcFrameConverter())

        pico.start()

        log.info("Started ...")
    }

    @Override
    void doGet(HttpServletRequest request, HttpServletResponse response) {
        def storage = pico.getComponent(PostgreSQLComponent)
        String html, json
        if (jobs) {
            List services = props.scheduledServices.split(",")
            def state = [:]
            StringBuilder table = new StringBuilder("<table cellspacing=\"10\"><tr><th>&nbsp;</th>")
            table.append("<form method=\"post\">")

            Set catSet = new TreeSet<String>()

            for (service in services) {
                state[service] = storage.loadSettings(SETTINGS_PFX+service)
                state[service]["harvesterClass"] = props.getProperty(service + ".harvesterClass", DEFAULT_HARVESTER)
                state[service]["serviceUrl"] = props.getProperty(service + ".serviceUrl")
                state[service]["interval"] = props.getProperty(service + ".interval", "" + DEFAULT_INTERVAL)
                catSet.add("harvesterClass")
                catSet.add("serviceUrl")
                catSet.add("interval")
                catSet.addAll(state[service].keySet())
                table.append("<th>$service</th>")

            }
            table.append("</tr>")
            List categories = catSet.toList()

            int i = 0
            for (cat in categories) {
                table.append("<tr><td align=\"right\"><b>$cat</b></td>")
                for (collection in services) {
                    table.append("<td>${state.get(collection).get(cat) != null ? state.get(collection).get(cat) : "&nbsp;"}</td>")
                }
                table.append("</tr>")
            }
            table.append("<tr><td><input type=\"submit\" name=\"action_all\" value=\"stop all\"></td>")
            for (collection in services) {
                table.append("<td><input type=\"submit\" name=\"action_${collection}\" value=\"${jobs[collection]?.active ? "stop" : "start"}\">")
                if (jobs[collection] && !jobs[collection].active) {
                    String lastImportDate = jobs[collection].getLastImportValue().format("yyyy-MM-dd'T'HH:mm")
                    table.append("&nbsp;<input type=\"submit\" name=\"reset_${collection}\" value=\"reload $collection from\"/>&nbsp;<input type=\"datetime-local\" name=\"datevalue\" value=\"${lastImportDate}\"/>")
                }
                table.append("</td>")
            }
            table.append("</tr>")

            table.append("</form></table>")

            html = """
                <html><head><title>OAIPMH Harvester control panel</title></head>
                <body>
                ${table.toString()}
                </form>
                """
            json = mapper.writeValueAsString(state)
        } else {

            html = """
                <html><head><title>OAIPMH Harvest Disabled</title></head>
                <body>

                HARVESTER DISABLED<br/>
                """
            json = mapper.writeValueAsString(["state":"disabled"])
        }
        PrintWriter out = response.getWriter();

        if (request.getPathInfo() == "/json") {
            response.setContentType("application/json");
            out.print(json);
        } else {
            response.setContentType("text/html");
            out.print(html);
        }

        out.flush();
    }

    void doPost(HttpServletRequest request, HttpServletResponse response) {
        log.debug("Received post request. Got this: ${request.getParameterMap()}")
        for (reqs in request.getParameterNames()) {
            if (reqs == "action_all") {
                for (job in jobs) {
                    job.value.disable()
                }
            } else if (reqs.startsWith("reset_")) {
                log.debug("Loading job for ${reqs.substring(6)}")
                log.debug("Got these jobs: $jobs")
                def job = jobs.get(reqs.substring(6))
                Date startDate = Date.parse("yyyy-MM-dd'T'HH:mm", request.getParameter("datevalue"))
                log.debug("Resetting harvester for ${job.collection} to $startDate")
                job.setStartDate(startDate)
                //job.enable()
            } else if (reqs.startsWith("action_")) {
                jobs.get(reqs.substring(7)).toggleActive()
            }
        }
        response.sendRedirect(request.getRequestURL().toString())
    }

    Map loadOaiPmhSettings() {
        Map settings = pico.getComponent(PostgreSQLComponent.class).loadSettings("oaipmh")

    }

    void init() {
        log.info("Initializing OAIPMH harvester.")
        PostgreSQLComponent storage = pico.getComponent(PostgreSQLComponent.class)
        List services = props.scheduledServices.split(",")

        ScheduledExecutorService ses = Executors.newScheduledThreadPool(services.size())

        for (service in services) {

            log.info("Setting up schedule for $service")
            int scheduleIntervalSeconds = props.getProperty(service + ".interval", "" + DEFAULT_INTERVAL) as int
            String harvesterClass = props.getProperty(service + ".harvesterClass", DEFAULT_HARVESTER)
            String serviceUrl = props.getProperty(service + ".serviceUrl")
            String username = props.getProperty(service + ".username")
            String password = props.getProperty(service + ".password")
            String sourceSystem = props.getProperty(service + ".sourceSystem", DEFAULT_SYSTEM)
            def job = new ScheduledJob(pico.getComponent(Class.forName(harvesterClass)), "${SETTINGS_PFX}${service}",
                    serviceUrl, username, password, sourceSystem, storage)
            jobs[service] = job

            try {
                ses.scheduleWithFixedDelay(job, scheduleDelaySeconds, scheduleIntervalSeconds, TimeUnit.SECONDS)
            } catch (RejectedExecutionException ree) {
                log.error("execution failed", ree)
            }
        }
        log.info("scheduler started")
    }
}

@Log
{'fraction_non_alphanumeric': 0.12491245941300057, 'fraction_numerical': 0.0024193034952568917, 'mean_word_length': 2.4132985658409387, 'pattern_counts': {'":': 2, '<': 91, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3229014', 'n_tokens_mistral': 4422, 'n_tokens_neox': 4172, 'n_words': 766}