<commit_before>package qa.qcri.aidr.trainer.pybossa.service.impl;

import au.com.bytecode.opencsv.CSVWriter;
import org.apache.log4j.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import qa.qcri.aidr.trainer.pybossa.entity.*;
import qa.qcri.aidr.trainer.pybossa.format.impl.CVSRemoteFileFormatter;
import qa.qcri.aidr.trainer.pybossa.format.impl.GeoJsonOutputModel;
import qa.qcri.aidr.trainer.pybossa.service.*;
import qa.qcri.aidr.trainer.pybossa.store.PybossaConf;
import qa.qcri.aidr.trainer.pybossa.store.StatusCodeType;
import qa.qcri.aidr.trainer.pybossa.store.URLPrefixCode;
import qa.qcri.aidr.trainer.pybossa.store.UserAccount;
import qa.qcri.aidr.trainer.pybossa.util.DateTimeConverter;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.Iterator;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: jlucas
 * Date: 11/22/13
 * Time: 2:29 PM
 * To change this template use File | Settings | File Templates.
 */
@Service("reportProductService")
@Transactional(readOnly = false)
public class ReportProductServiceImpl implements ReportProductService {

    protected static Logger logger = Logger.getLogger(ReportProductServiceImpl.class);

    @Autowired
    private ClientService clientService;

    @Autowired
    private ClientAppService clientAppService;

    @Autowired
    private ReportTemplateService reportTemplateService;

    @Autowired
    private ClientAppSourceService clientAppSourceService;

    @Autowired
    private ClientAppEventService clientAppEventService;

    private Client client;
    private PybossaCommunicator pybossaCommunicator = new PybossaCommunicator();

    public void setClassVariable() throws Exception{
        client = clientService.findClientByCriteria("name", UserAccount.SYSTEM_USER_NAME);
    }


    @Override
    public void generateCVSReportForGeoClicker() throws Exception{
        setClassVariable();
        if(client == null){
            return;
        }
        List<ClientApp> appList = clientAppService.getAllClientAppByClientID(client.getClientID() );
       // System.out.println("appList size: " + appList.size() + " - " + client.getClientID());
        Iterator itr= appList.iterator();

        while(itr.hasNext()){
            ClientApp clientApp = (ClientApp)itr.next();
            List<ReportTemplate> templateList =  reportTemplateService.getReportTemplateByClientApp(clientApp.getClientAppID(), StatusCodeType.TEMPLATE_IS_READY_FOR_EXPORT);

            if(templateList.size() > StatusCodeType.MIN_REPORT_TEMPLATE_EXPORT_SIZE){
                CVSRemoteFileFormatter formatter = new CVSRemoteFileFormatter();
                String sTemp = DateTimeConverter.reformattedCurrentDateForFileName() + clientApp.getShortName() + "export.csv";
                String fileName = PybossaConf.DEFAULT_TRAINER_FILE_PATH + sTemp;
                String mmFetchFileName = URLPrefixCode.MM_GEO_SOURCE_PATH +sTemp;
                CSVWriter writer = formatter.instanceToOutput(fileName);

                for(int i=0; i < templateList.size(); i++){
                    ReportTemplate rpt = templateList.get(i);
                    String answer = rpt.getAnswer().trim().toLowerCase();

                    formatter.addToCVSOuputFile(generateOutputData(rpt),writer);

                    rpt.setStatus(StatusCodeType.TEMPLATE_EXPORTED);
                    reportTemplateService.updateReportItem(rpt);
                }
                formatter.finalizeCVSOutputFile(writer);
                ClientAppEvent targetClinetApp = clientAppEventService.getNextSequenceClientAppEvent(clientApp.getClientAppID());
                ClientAppSource appSource = new ClientAppSource(targetClinetApp.getClientAppID(), StatusCodeType.EXTERNAL_DATA_SOURCE_ACTIVE, fileName);
                clientAppSourceService.insertNewClientAppSource(appSource);

                JSONArray jsonArray = new JSONArray();
                JSONObject obj= new JSONObject();
                obj.put("fileURL",mmFetchFileName);
                obj.put("appID",clientApp.getClientAppID());

                jsonArray.add(obj);

                String returnValue = pybossaCommunicator.sendPostGet(jsonArray.toJSONString(), URLPrefixCode.MICROMAPPER_API_SOURCE_SAVE_URL);

                //System.out.println("generateCVSReportForGeoClicker returnValue :" + returnValue);

            }
            // insert into source for file
        }
        //To change body of implemented methods use File | Settings | File Templates.
    }

    private String[] generateOutputData(ReportTemplate rpt){

        String[] data = new String[8];
        data[0] = rpt.getTweetID().toString();
        data[1] = rpt.getTweet();
        data[2] = rpt.getAuthor();
        data[3] = rpt.getLat();
        data[4] = rpt.getLon();
        data[5] = rpt.getUrl();
        data[6] = rpt.getCreated();
        data[7] = rpt.getAnswer();

        return data;
    }
}<commit_msg>add back code<commit_after>package qa.qcri.aidr.trainer.pybossa.service.impl;

import au.com.bytecode.opencsv.CSVWriter;
import org.apache.log4j.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import qa.qcri.aidr.trainer.pybossa.entity.*;
import qa.qcri.aidr.trainer.pybossa.format.impl.CVSRemoteFileFormatter;
import qa.qcri.aidr.trainer.pybossa.format.impl.GeoJsonOutputModel;
import qa.qcri.aidr.trainer.pybossa.service.*;
import qa.qcri.aidr.trainer.pybossa.store.PybossaConf;
import qa.qcri.aidr.trainer.pybossa.store.StatusCodeType;
import qa.qcri.aidr.trainer.pybossa.store.URLPrefixCode;
import qa.qcri.aidr.trainer.pybossa.store.UserAccount;
import qa.qcri.aidr.trainer.pybossa.util.DateTimeConverter;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.Iterator;
import java.util.List;

/**
 * Created with IntelliJ IDEA.
 * User: jlucas
 * Date: 11/22/13
 * Time: 2:29 PM
 * To change this template use File | Settings | File Templates.
 */
@Service("reportProductService")
@Transactional(readOnly = false)
public class ReportProductServiceImpl implements ReportProductService {

    protected static Logger logger = Logger.getLogger(ReportProductServiceImpl.class);

    @Autowired
    private ClientService clientService;

    @Autowired
    private ClientAppService clientAppService;

    @Autowired
    private ReportTemplateService reportTemplateService;

    @Autowired
    private ClientAppSourceService clientAppSourceService;

    @Autowired
    private ClientAppEventService clientAppEventService;

    private Client client;
    private PybossaCommunicator pybossaCommunicator = new PybossaCommunicator();

    public void setClassVariable() throws Exception{
        client = clientService.findClientByCriteria("name", UserAccount.SYSTEM_USER_NAME);
    }


    @Override
    public void generateCVSReportForGeoClicker() throws Exception{
        setClassVariable();
        if(client == null){
            return;
        }
        List<ClientApp> appList = clientAppService.getAllClientAppByClientID(client.getClientID() );
       // System.out.println("appList size: " + appList.size() + " - " + client.getClientID());
        Iterator itr= appList.iterator();

        while(itr.hasNext()){
            ClientApp clientApp = (ClientApp)itr.next();
            List<ReportTemplate> templateList =  reportTemplateService.getReportTemplateByClientApp(clientApp.getClientAppID(), StatusCodeType.TEMPLATE_IS_READY_FOR_EXPORT);

            if(templateList.size() > StatusCodeType.MIN_REPORT_TEMPLATE_EXPORT_SIZE){
                CVSRemoteFileFormatter formatter = new CVSRemoteFileFormatter();
                String sTemp = DateTimeConverter.reformattedCurrentDateForFileName() + clientApp.getShortName() + "export.csv";
                String fileName = PybossaConf.DEFAULT_TRAINER_FILE_PATH + sTemp;
                String mmFetchFileName = URLPrefixCode.MM_GEO_SOURCE_PATH +sTemp;
                CSVWriter writer = formatter.instanceToOutput(fileName);

                for(int i=0; i < templateList.size(); i++){
                    ReportTemplate rpt = templateList.get(i);
                    String answer = rpt.getAnswer().trim().toLowerCase();

                    formatter.addToCVSOuputFile(generateOutputData(rpt),writer);

                    rpt.setStatus(StatusCodeType.TEMPLATE_EXPORTED);
                    reportTemplateService.updateReportItem(rpt);
                }
                formatter.finalizeCVSOutputFile(writer);
                ClientAppEvent targetClinetApp = clientAppEventService.getNextSequenceClientAppEvent(clientApp.getClientAppID());
                ClientAppSource appSource = new ClientAppSource(targetClinetApp.getClientAppID(), StatusCodeType.EXTERNAL_DATA_SOURCE_ACTIVE, fileName);
                clientAppSourceService.insertNewClientAppSource(appSource);

                JSONArray jsonArray = new JSONArray();
                JSONObject obj= new JSONObject();
                obj.put("fileURL",mmFetchFileName);
                obj.put("appID",targetClinetApp.getClientAppID());

                jsonArray.add(obj);

                String returnValue = pybossaCommunicator.sendPostGet(jsonArray.toJSONString(), URLPrefixCode.MICROMAPPER_API_SOURCE_SAVE_URL);

            }

        }

    }

    private String[] generateOutputData(ReportTemplate rpt){

        String[] data = new String[8];
        data[0] = rpt.getTweetID().toString();
        data[1] = rpt.getTweet();
        data[2] = rpt.getAuthor();
        data[3] = rpt.getLat();
        data[4] = rpt.getLon();
        data[5] = rpt.getUrl();
        data[6] = rpt.getCreated();
        data[7] = rpt.getAnswer();

        return data;
    }
}
{'fraction_non_alphanumeric': 0.09880649579338681, 'fraction_numerical': 0.003913128546272745, 'mean_word_length': 3.2899706252622742, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7493405', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2939, 'n_words': 487}