<commit_before>/*
 *  Copyright (C) 2020 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.akvo.gae.remoteapi;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


public class FixCopyDependency implements Process {

    @Override
    public void execute(DatastoreService ds, String[] args) throws Exception {
        fixDependencyForSurvey(ds, 107750251L);
    }

    private void fixDependencyForSurvey(DatastoreService ds, long surveyId) {

        final Query q = new Query("Question");
        final Filter surveyFilter = new FilterPredicate("surveyId", FilterOperator.EQUAL, surveyId);
        final Filter iamDependent = new FilterPredicate("dependentFlag", FilterOperator.EQUAL, true);
        final Filter idontHaveDependentQuestionId = new FilterPredicate("dependentQuestionId", FilterOperator.EQUAL, null);

        final Filter theFilter = new Query.CompositeFilter(Query.CompositeFilterOperator.AND, Arrays.asList(surveyFilter, iamDependent, idontHaveDependentQuestionId));
        q.setFilter(theFilter);

        final PreparedQuery pq = ds.prepare(q);
<commit_msg>[#3689] Adds a function to get candidate forms to be fixed<commit_after>/*
 *  Copyright (C) 2020 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.akvo.gae.remoteapi;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;

import java.sql.Date;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;


public class FixCopyDependency implements Process {

    @Override
    public void execute(DatastoreService ds, String[] args) throws Exception {
        //fixDependencyForSurvey(ds, 107750251L);
        System.out.println(surveysWithMissingDependencies(ds));
    }

    private Set<Long> surveysWithMissingDependencies(DatastoreService ds) {
        Set<Long> brokenSurveys = new HashSet<>();

        Instant t = Instant.parse("2020-11-01T00:00:00Z");

        Query q = new Query("Question");
        Filter f1 = new FilterPredicate("createdDateTime", FilterOperator.GREATER_THAN_OR_EQUAL, Date.from(t));
        q.setFilter(f1);

        PreparedQuery pq = ds.prepare(q);

        for (Entity question : pq.asIterable(FetchOptions.Builder.withChunkSize(500))) {
            if (Boolean.TRUE.equals(question.getProperty("dependentFlag")) && question.getProperty("dependentQuestionId") == null) {
                brokenSurveys.add((Long) question.getProperty("surveyId"));
            }
        }

        return brokenSurveys;
    }

    private void fixDependencyForSurvey(DatastoreService ds, long surveyId) {

        final Query q = new Query("Question");
        final Filter surveyFilter = new FilterPredicate("surveyId", FilterOperator.EQUAL, surveyId);
        final Filter iamDependent = new FilterPredicate("dependentFlag", FilterOperator.EQUAL, true);
        final Filter idontHaveDependentQuestionId = new FilterPredicate("dependentQuestionId", FilterOperator.EQUAL, null);

        final Filter theFilter = new Query.CompositeFilter(Query.CompositeFilterOperator.AND, Arrays.asList(surveyFilter, iamDependent, idontHaveDependentQuestionId));
        q.setFilter(theFilter);

        final PreparedQuery pq = ds.prepare(q);

{'fraction_non_alphanumeric': 0.09178490103345595, 'fraction_numerical': 0.008933263268523384, 'mean_word_length': 4.282146160962072, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1692506', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1621, 'n_words': 452}