<commit_before>/*
 *  Copyright (C) 2018 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.akvo.gae.remoteapi;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;

import java.util.ArrayList;
import java.util.List;

/*
 * - Checks that the DataPoint (SurveyedLocale) location is correct and updates it using data from
 * the localeGeolocation field of each SurveyInstance.
 *
 * This may take a very long time so it is better to provide the surveyId (the id of SurveyGroup)
 */
public class CheckDataPointLocation implements Process {

    private static final long MISSING_SURVEY_ID = -1;

    @Override
    public void execute(DatastoreService ds, String[] args) {
        long timeStart = System.currentTimeMillis();
        System.out.println(
                "#Arguments: survey id to fix one survey, FIX to correct data point location");

        boolean fixDataPointLocation = dataPointFixRequested(args);
        long surveyId = getRequestedSurveyId(args);

        List<Entity> dataPointsToFix = getDataToFix(ds, surveyId);

        long timeEnd = System.currentTimeMillis();
        System.out.println("Getting data to fix took: " + (timeEnd - timeStart) + " ms");
        System.out.println(dataPointsToFix.size() + " data points need update");

        fixDataPoints(ds, timeStart, fixDataPointLocation, dataPointsToFix);
    }

    private void fixDataPoints(DatastoreService ds, long timeStart, boolean fixDataPointLocation,
            List<Entity> dataPointsToFix) {
        if (fixDataPointLocation) {
            if (!dataPointsToFix.isEmpty()) {
                System.out.println("Will fix data...");
                DataUtils.batchSaveEntities(ds, dataPointsToFix);
                long timeEnd = System.currentTimeMillis();
                System.out.println("Fixing data took: " + (timeEnd - timeStart) + " ms");
            } else {
                System.out.println("No data to fix...");
            }
        }
    }

    private long getRequestedSurveyId(String[] args) {
        long surveyId = MISSING_SURVEY_ID;
        if (args == null) {
            return surveyId;
        }
        for (String arg : args) {
            surveyId = safeParseSurveyId(arg);
            if (surveyId != MISSING_SURVEY_ID) {
                return surveyId;
            }
        }
        return surveyId;
    }

    private boolean dataPointFixRequested(String[] args) {
        if (args == null) {
            return false;
        }
        for (String arg : args) {
            if ("FIX".equalsIgnoreCase(arg)) {
                return true;
            }
        }
        return false;
    }

    private long safeParseSurveyId(String longAsString) {
        if (longAsString == null || longAsString.isEmpty()) {
            return MISSING_SURVEY_ID;
        }
        try {
            return Long.parseLong(longAsString);
        } catch (NumberFormatException e) {
            return MISSING_SURVEY_ID;
        }
    }

    private List<Entity> getDataToFix(DatastoreService ds, long surveyId) {
        List<Entity> brokenDataPoints = new ArrayList<>();
        if (surveyId == MISSING_SURVEY_ID) {
            Iterable<Entity> entities = getSurveys(ds);
            int surveyCounter = 0;
            for (Entity survey : entities) {
                surveyCounter++;
                brokenDataPoints.addAll(getDataPointsToFixForSurvey(ds, survey));
            }
            System.out.println("Found " + surveyCounter + " monitored SurveyGroups to check");
        } else {
            Key key = KeyFactory.createKey("SurveyGroup", surveyId);
            try {
                System.out.println("Checking data points for one survey: "+surveyId);
                Entity entity = ds.get(key);
                brokenDataPoints = getDataPointsToFixForSurvey(ds, entity);
            } catch (EntityNotFoundException e) {
                e.printStackTrace();
            }
        }
        return brokenDataPoints;
    }

    private List<Entity> getDataPointsToFixForSurvey(DatastoreService ds, Entity survey) {
        List<Entity> brokenDataPoints = new ArrayList<>();
        long surveyId = survey.getKey().getId();
        Long registrationFormId = (Long) survey.getProperty("newLocaleSurveyId");
        if (registrationFormId != null) {
            List<Entity> list = getForms(ds, surveyId);
            if (list.size() > 1) {
                Entity geoQuestion = getGeoQuestion(ds, registrationFormId);
                if (geoQuestion != null) {
                    List<Entity> dataPoints = getDataPoints(ds, surveyId);
                    if (dataPoints.size() > 0) {
                        long geoQuestionId = geoQuestion.getKey().getId();
                        for (Entity dataPoint : dataPoints) {
                            Entity dataPointToUpdate = getDataPointToUpdate(ds, dataPoint,
                                    registrationFormId, geoQuestionId);
                            if (dataPointToUpdate != null) {
                                brokenDataPoints.add(dataPointToUpdate);
                            }
                        }
                    }
                }
            }
        }
        return brokenDataPoints;
    }

    private Entity getDataPointToUpdate(DatastoreService ds, Entity dataPoint,
            Long registrationFormId, long geoQuestionId) {
        long dataPointId = dataPoint.getKey().getId();
        long surveyInstanceId = getSurveyInstanceId(ds, registrationFormId,
                (String) dataPoint.getProperty("identifier"), dataPointId);
        if (surveyInstanceId != -1) {
            Entity questionAnswer = getQuestionAnswer(ds, surveyInstanceId, geoQuestionId);
            if (questionAnswer != null) {
                Double dataPointLatitude = (Double) dataPoint.getProperty("latitude");
                Double dataPointLongitude = (Double) dataPoint.getProperty("longitude");

                String answerValue = (String) questionAnswer.getProperty("value");
                if (answerValue != null && !answerValue.isEmpty()) {
                    String[] answerBits = answerValue.split("\\|");
                    if (answerBits.length > 1) {
                        Double answerLatitude = safeParseDouble(answerBits[0]);
                        Double answerLongitude = safeParseDouble(answerBits[1]);

                        boolean dataPointLocationNeedsUpdate =
                                answerLatitude != null && answerLongitude != null
                                        && (!answerLatitude.equals(dataPointLatitude)
                                        || !answerLongitude.equals(dataPointLongitude));
                        if (dataPointLocationNeedsUpdate) {
                            dataPoint.setProperty("latitude", answerLatitude);
                            dataPoint.setProperty("longitude", answerLongitude);

                            System.out.println("Data point " + dataPointId + " needs fixing");
                            return dataPoint;
                        }
                    }
                }
            }
        }
        return null;<commit_msg>[#2520] Add geocells to the datapoint when fixing location<commit_after>/*
 *  Copyright (C) 2018 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.akvo.gae.remoteapi;

import com.beoui.geocell.GeocellManager;
import com.beoui.geocell.model.Point;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;

import java.util.ArrayList;
import java.util.List;

/*
 * - Checks that the DataPoint (SurveyedLocale) location is correct and updates it using data from
 * the localeGeolocation field of each SurveyInstance.
 *
 * This may take a very long time so it is better to provide the surveyId (the id of SurveyGroup)
 */
public class CheckDataPointLocation implements Process {

    private static final long MISSING_SURVEY_ID = -1;

    @Override
    public void execute(DatastoreService ds, String[] args) {
        long timeStart = System.currentTimeMillis();
        System.out.println(
                "#Arguments: survey id to fix one survey, FIX to correct data point location");

        boolean fixDataPointLocation = dataPointFixRequested(args);
        long surveyId = getRequestedSurveyId(args);

        List<Entity> dataPointsToFix = getDataToFix(ds, surveyId);

        long timeEnd = System.currentTimeMillis();
        System.out.println("Getting data to fix took: " + (timeEnd - timeStart) + " ms");
        System.out.println(dataPointsToFix.size() + " data points need update");

        fixDataPoints(ds, timeStart, fixDataPointLocation, dataPointsToFix);
    }

    private void fixDataPoints(DatastoreService ds, long timeStart, boolean fixDataPointLocation,
            List<Entity> dataPointsToFix) {
        if (fixDataPointLocation) {
            if (!dataPointsToFix.isEmpty()) {
                System.out.println("Will fix data...");
                DataUtils.batchSaveEntities(ds, dataPointsToFix);
                long timeEnd = System.currentTimeMillis();
                System.out.println("Fixing data took: " + (timeEnd - timeStart) + " ms");
            } else {
                System.out.println("No data to fix...");
            }
        }
    }

    private long getRequestedSurveyId(String[] args) {
        long surveyId = MISSING_SURVEY_ID;
        if (args == null) {
            return surveyId;
        }
        for (String arg : args) {
            surveyId = safeParseSurveyId(arg);
            if (surveyId != MISSING_SURVEY_ID) {
                return surveyId;
            }
        }
        return surveyId;
    }

    private boolean dataPointFixRequested(String[] args) {
        if (args == null) {
            return false;
        }
        for (String arg : args) {
            if ("FIX".equalsIgnoreCase(arg)) {
                return true;
            }
        }
        return false;
    }

    private long safeParseSurveyId(String longAsString) {
        if (longAsString == null || longAsString.isEmpty()) {
            return MISSING_SURVEY_ID;
        }
        try {
            return Long.parseLong(longAsString);
        } catch (NumberFormatException e) {
            return MISSING_SURVEY_ID;
        }
    }

    private List<Entity> getDataToFix(DatastoreService ds, long surveyId) {
        List<Entity> brokenDataPoints = new ArrayList<>();
        if (surveyId == MISSING_SURVEY_ID) {
            Iterable<Entity> entities = getSurveys(ds);
            int surveyCounter = 0;
            for (Entity survey : entities) {
                surveyCounter++;
                brokenDataPoints.addAll(getDataPointsToFixForSurvey(ds, survey));
            }
            System.out.println("Found " + surveyCounter + " monitored SurveyGroups to check");
        } else {
            Key key = KeyFactory.createKey("SurveyGroup", surveyId);
            try {
                System.out.println("Checking data points for one survey: "+surveyId);
                Entity entity = ds.get(key);
                brokenDataPoints = getDataPointsToFixForSurvey(ds, entity);
            } catch (EntityNotFoundException e) {
                e.printStackTrace();
            }
        }
        return brokenDataPoints;
    }

    private List<Entity> getDataPointsToFixForSurvey(DatastoreService ds, Entity survey) {
        List<Entity> brokenDataPoints = new ArrayList<>();
        long surveyId = survey.getKey().getId();
        Long registrationFormId = (Long) survey.getProperty("newLocaleSurveyId");
        if (registrationFormId != null) {
            List<Entity> list = getForms(ds, surveyId);
            if (list.size() > 0) {
                Entity geoQuestion = getGeoQuestion(ds, registrationFormId);
                if (geoQuestion != null) {
                    List<Entity> dataPoints = getDataPoints(ds, surveyId);
                    if (dataPoints.size() > 0) {
                        long geoQuestionId = geoQuestion.getKey().getId();
                        for (Entity dataPoint : dataPoints) {
                            Entity dataPointToUpdate = getDataPointToUpdate(ds, dataPoint,
                                    registrationFormId, geoQuestionId);
                            if (dataPointToUpdate != null) {
                                brokenDataPoints.add(dataPointToUpdate);
                            }
                        }
                    }
                }
            }
        }
        return brokenDataPoints;
    }

    private Entity getDataPointToUpdate(DatastoreService ds, Entity dataPoint,
            Long registrationFormId, long geoQuestionId) {
        long dataPointId = dataPoint.getKey().getId();
        long surveyInstanceId = getSurveyInstanceId(ds, registrationFormId,
                (String) dataPoint.getProperty("identifier"), dataPointId);
        if (surveyInstanceId != -1) {
            Entity questionAnswer = getQuestionAnswer(ds, surveyInstanceId, geoQuestionId);
            if (questionAnswer != null) {
                Double dataPointLatitude = (Double) dataPoint.getProperty("latitude");
                Double dataPointLongitude = (Double) dataPoint.getProperty("longitude");
                List<String> geocells = (List<String>) dataPoint.getProperty("geocells");

                String answerValue = (String) questionAnswer.getProperty("value");
                if (answerValue != null && !answerValue.isEmpty()) {
                    String[] answerBits = answerValue.split("\\|");
                    if (answerBits.length > 1) {
                        Double answerLatitude = safeParseDouble(answerBits[0]);
                        Double answerLongitude = safeParseDouble(answerBits[1]);

                        boolean dataPointLocationNeedsUpdate = answerLatitude != null
                                        && answerLongitude != null
                                        && (!answerLatitude.equals(dataPointLatitude)
                                            || !answerLongitude.equals(dataPointLongitude)
                                            || geocells == null
                                            || geocells.isEmpty());
                        if (dataPointLocationNeedsUpdate) {
                            dataPoint.setProperty("latitude", answerLatitude);
                            dataPoint.setProperty("longitude", answerLongitude);
                            dataPoint.setProperty("geocells", GeocellManager.generateGeoCell(new Point(answerLatitude, answerLongitude)));

                            System.out.println("Data point " + dataPointId + " needs fixing");
                            return dataPoint;
                        }
                    }
                }
            }
        }
        return null;
{'fraction_non_alphanumeric': 0.08977892452940832, 'fraction_numerical': 0.0017592212513927168, 'mean_word_length': 2.037221727515583, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5107724', 'n_tokens_mistral': 4431, 'n_tokens_neox': 4303, 'n_words': 1080}