<commit_before>/*******************************************************************************
 * Copyright Euresearch - 2014 - http://www.euresearch.ch
 * Proprietary software license.
 *******************************************************************************/

package com.searchbox.collection.oppfin;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.log4j.lf5.util.DateFormatManager;
import org.jsoup.Jsoup;
import org.jsoup.examples.HtmlToPlainText;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.job.builder.FlowJobBuilder;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.searchbox.collection.AbstractBatchCollection;
import com.searchbox.collection.StandardCollection;
import com.searchbox.collection.SynchronizedCollection;
import com.searchbox.core.dm.Field;
import com.searchbox.core.dm.FieldMap;

@Configurable
@Component
public class IdealISTCollection extends AbstractBatchCollection implements
    SynchronizedCollection, StandardCollection, InitializingBean {

  private static final Logger LOGGER = LoggerFactory
      .getLogger(IdealISTCollection.class);

  private final static String CRAWLER_USER_AGENT = "crawler.userAgent";
  private final static String IDEALIST_LIST_SERVICE = "idealist.service.list.url";
  private final static String IDEALIST_DOCUMENT_SERVICE = "idealist.service.document.url";

  private final static String CRAWLER_USER_AGENT_DEFAULT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11";
  private final static String IDEALIST_LIST_SERVICE_DEFAULT = "http://www.ideal-ist.eu/idealist_oppfinder/documents";
  private final static String IDEALIST_DOCUMENT_SERVICE_DEFAULT = "http://www.ideal-ist.eu/idealist_oppfinder/document";

  DocumentBuilder db;

  DateFormat dfmt = new DateFormatManager("dd/MM/yyyy").getDateFormatInstance();

  public IdealISTCollection() {
    super("Idealist");
  }

  public static List<Field> GET_FIELDS() {
    List<Field> fields = new ArrayList<Field>();
    fields.add(new Field(String.class, "uid"));
    fields.add(new Field(String.class, "docSource"));
    fields.add(new Field(String.class, "docType"));
    fields.add(new Field(String.class, "programme"));

    fields.add(new Field(String.class, "idealistTitle"));
    fields.add(new Field(String.class, "idealistPsId"));
    fields.add(new Field(String.class, "idealistStatus"));
    
    fields.add(new Field(Date.class, "idealistDeadline"));
    fields.add(new Field(Date.class, "idealistUpdated"));
    fields.add(new Field(Date.class, "idealistPublished"));
    
    fields.add(new Field(String.class, "idealistObjective"));
    fields.add(new Field(String.class, "idealistFundingScheme"));
    fields.add(new Field(String.class, "idealistEvaluationScheme"));
    fields.add(new Field(String.class, "idealistTypeOfPartnerSought"));
    fields.add(new Field(String.class, "idealistCoordinationPossible"));
    fields.add(new Field(String.class, "idealistOrganisation"));
    fields.add(new Field(String.class, "idealistDepartement"));
    fields.add(new Field(String.class, "idealistTypeOfOrganisation"));
    fields.add(new Field(String.class, "idealistCountry"));
    fields.add(new Field(String.class, "idealistBody"));
    fields.add(new Field(String.class, "idealistOutline"));
    fields.add(new Field(String.class, "idealistDescriptionOfWork"));

    fields.add(new Field(String.class, "callIdentifier"));

    return fields;
  }
  
  @Override
  public String getIdValue(FieldMap fields) {
    return (String) fields.get(this.getIdFieldName()).get(0);
  }

  @Override
  public String getBodyValue(FieldMap fields) {
        return (String) fields.get("idealistBody").get(0);
  }

  @Override
  public String getTitleValue(FieldMap fields) {
    return (String) fields.get("idealistTitle").get(0);
  }
  
  @Override
  public Date getPublishedValue(FieldMap fields) {
        return null;
  }

  @Override<commit_msg>Adding test on idealist crawler (avoid nullpointer exceptions).<commit_after>/*******************************************************************************
 * Copyright Euresearch - 2014 - http://www.euresearch.ch
 * Proprietary software license.
 *******************************************************************************/

package com.searchbox.collection.oppfin;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.log4j.lf5.util.DateFormatManager;
import org.jsoup.Jsoup;
import org.jsoup.examples.HtmlToPlainText;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.job.builder.FlowJobBuilder;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import com.searchbox.collection.AbstractBatchCollection;
import com.searchbox.collection.StandardCollection;
import com.searchbox.collection.SynchronizedCollection;
import com.searchbox.core.dm.Field;
import com.searchbox.core.dm.FieldMap;

@Configurable
@Component
public class IdealISTCollection extends AbstractBatchCollection implements
    SynchronizedCollection, StandardCollection, InitializingBean {

  private static final Logger LOGGER = LoggerFactory
      .getLogger(IdealISTCollection.class);

  private final static String CRAWLER_USER_AGENT = "crawler.userAgent";
  private final static String IDEALIST_LIST_SERVICE = "idealist.service.list.url";
  private final static String IDEALIST_DOCUMENT_SERVICE = "idealist.service.document.url";

  private final static String CRAWLER_USER_AGENT_DEFAULT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11";
  private final static String IDEALIST_LIST_SERVICE_DEFAULT = "http://www.ideal-ist.eu/idealist_oppfinder/documents";
  private final static String IDEALIST_DOCUMENT_SERVICE_DEFAULT = "http://www.ideal-ist.eu/idealist_oppfinder/document";

  DocumentBuilder db;

  DateFormat dfmt = new DateFormatManager("dd/MM/yyyy").getDateFormatInstance();

  public IdealISTCollection() {
    super("Idealist");
  }

  public static List<Field> GET_FIELDS() {
    List<Field> fields = new ArrayList<Field>();
    fields.add(new Field(String.class, "uid"));
    fields.add(new Field(String.class, "docSource"));
    fields.add(new Field(String.class, "docType"));
    fields.add(new Field(String.class, "programme"));

    fields.add(new Field(String.class, "idealistTitle"));
    fields.add(new Field(String.class, "idealistPsId"));
    fields.add(new Field(String.class, "idealistStatus"));
    
    fields.add(new Field(Date.class, "idealistDeadline"));
    fields.add(new Field(Date.class, "idealistUpdated"));
    fields.add(new Field(Date.class, "idealistPublished"));
    
    fields.add(new Field(String.class, "idealistObjective"));
    fields.add(new Field(String.class, "idealistFundingScheme"));
    fields.add(new Field(String.class, "idealistEvaluationScheme"));
    fields.add(new Field(String.class, "idealistTypeOfPartnerSought"));
    fields.add(new Field(String.class, "idealistCoordinationPossible"));
    fields.add(new Field(String.class, "idealistOrganisation"));
    fields.add(new Field(String.class, "idealistDepartement"));
    fields.add(new Field(String.class, "idealistTypeOfOrganisation"));
    fields.add(new Field(String.class, "idealistCountry"));
    fields.add(new Field(String.class, "idealistBody"));
    fields.add(new Field(String.class, "idealistOutline"));
    fields.add(new Field(String.class, "idealistDescriptionOfWork"));

    fields.add(new Field(String.class, "callIdentifier"));

    return fields;
  }
  
  @Override
  public String getIdValue(FieldMap fields) {
    return (fields.get(this.getIdFieldName()).size() > 0) ?
    		(String) fields.get(this.getIdFieldName()).get(0) :
    		null;
  }

  @Override
  public String getBodyValue(FieldMap fields) {
        return (fields.get("idealistBody").size() > 0)  ? 
        		(String) fields.get("idealistBody").get(0) : 
        		null;
  }

  @Override
  public String getTitleValue(FieldMap fields) {
    return (fields.get("idealistTitle").size() > 0) ? 
    		(String) fields.get("idealistTitle").get(0) : 
    		null;
  }
  
  @Override
  public Date getPublishedValue(FieldMap fields) {
        return null;
  }

  @Override
{'fraction_non_alphanumeric': 0.14181209236735412, 'fraction_numerical': 0.006610259122157589, 'mean_word_length': 5.670781893004115, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7151691', 'n_tokens_mistral': 3491, 'n_tokens_neox': 3328, 'n_words': 599}