<commit_before>/*
 * CODENVY CONFIDENTIAL
 * __________________
 *
 *  [2012] - [2015] Codenvy, S.A.
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of Codenvy S.A. and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to Codenvy S.A.
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Codenvy S.A..
 */

package com.codenvy.analytics.services.integrity;

import com.codenvy.analytics.Utils;
import com.codenvy.analytics.metrics.AbstractMetric;
import com.codenvy.analytics.metrics.Context;
import com.codenvy.analytics.metrics.MetricType;
import com.codenvy.analytics.persistent.CollectionsManagement;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;

import javax.annotation.Nullable;
import java.io.IOException;
import java.text.ParseException;
import java.util.UUID;

import static com.codenvy.analytics.metrics.AbstractMetric.DATE;
import static com.codenvy.analytics.metrics.AbstractMetric.ID;
import static com.codenvy.analytics.metrics.AbstractMetric.MEMORY;
import static com.codenvy.analytics.metrics.AbstractMetric.PERSISTENT_WS;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT_ID;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT_TYPE;
import static com.codenvy.analytics.metrics.AbstractMetric.REGISTERED_USER;
import static com.codenvy.analytics.metrics.AbstractMetric.STOP_TIME;
import static com.codenvy.analytics.metrics.AbstractMetric.TIMEOUT;
import static com.codenvy.analytics.metrics.AbstractMetric.USER;
import static com.codenvy.analytics.metrics.AbstractMetric.WS;
import static com.codenvy.analytics.pig.udf.CalculateGigabyteRamHours.calculateGigabyteRamHours;

/**
 * @author <NAME>
 */
@Singleton
public class TasksIntegrity implements CollectionDataIntegrity {

    private final CollectionsManagement collectionsManagement;

    @Inject
    public TasksIntegrity(CollectionsManagement collectionsManagement) {
        this.collectionsManagement = collectionsManagement;
    }

    /** {@inheritDoc} */
    @Override
    public void doCompute(Context context) throws IOException {
        try {
            removeFinishedEventIfStartIsAbsent(context);
            updateUsageTime(context);
        } catch (ParseException e) {
            throw new IOException(e);
        }
    }

    private void updateUsageTime(Context context) throws ParseException {
        removeObsoleteData(context);

        DBCollection collection = collectionsManagement.getOrCreate(MetricType.TASKS.toString().toLowerCase());

        DBObject q = Utils.setDateFilter(context, STOP_TIME);
        q.put(AbstractMetric.TASK_TYPE, new BasicDBObject("$in", new String[]{"builder", "runner", "debugger"}));
        q.put(AbstractMetric.START_TIME, new BasicDBObject("$exists", true));

        DBCursor cursor = collection.find(q);
        while (cursor.hasNext()) {
            DBObject doc = cursor.next();

            String taskType = doc.get(AbstractMetric.TASK_TYPE).toString();
            String launchType = doc.get(AbstractMetric.LAUNCH_TYPE).toString();
            long startTime = Long.parseLong(doc.get(AbstractMetric.START_TIME).toString());
            long endTime = Long.parseLong(doc.get(AbstractMetric.STOP_TIME).toString());

            long timeout;<commit_msg>DASHB-621: gigabyte ram hour calculations are not accurate - bad times<commit_after>/*
 * CODENVY CONFIDENTIAL
 * __________________
 *
 *  [2012] - [2015] Codenvy, S.A.
 *  All Rights Reserved.
 *
 * NOTICE:  All information contained herein is, and remains
 * the property of Codenvy S.A. and its suppliers,
 * if any.  The intellectual and technical concepts contained
 * herein are proprietary to Codenvy S.A.
 * and its suppliers and may be covered by U.S. and Foreign Patents,
 * patents in process, and are protected by trade secret or copyright law.
 * Dissemination of this information or reproduction of this material
 * is strictly forbidden unless prior written permission is obtained
 * from Codenvy S.A..
 */

package com.codenvy.analytics.services.integrity;

import com.codenvy.analytics.Utils;
import com.codenvy.analytics.metrics.AbstractMetric;
import com.codenvy.analytics.metrics.Context;
import com.codenvy.analytics.metrics.MetricType;
import com.codenvy.analytics.persistent.CollectionsManagement;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;

import javax.annotation.Nullable;
import java.io.IOException;
import java.text.ParseException;
import java.util.UUID;

import static com.codenvy.analytics.metrics.AbstractMetric.DATE;
import static com.codenvy.analytics.metrics.AbstractMetric.ID;
import static com.codenvy.analytics.metrics.AbstractMetric.LAUNCH_TYPE;
import static com.codenvy.analytics.metrics.AbstractMetric.MEMORY;
import static com.codenvy.analytics.metrics.AbstractMetric.PERSISTENT_WS;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT_ID;
import static com.codenvy.analytics.metrics.AbstractMetric.PROJECT_TYPE;
import static com.codenvy.analytics.metrics.AbstractMetric.REGISTERED_USER;
import static com.codenvy.analytics.metrics.AbstractMetric.STOP_TIME;
import static com.codenvy.analytics.metrics.AbstractMetric.TIMEOUT;
import static com.codenvy.analytics.metrics.AbstractMetric.USER;
import static com.codenvy.analytics.metrics.AbstractMetric.WS;
import static com.codenvy.analytics.pig.udf.CalculateGigabyteRamHours.calculateGigabyteRamHours;

/**
 * @author <NAME>
 */
@Singleton
public class TasksIntegrity implements CollectionDataIntegrity {

    private final CollectionsManagement collectionsManagement;

    @Inject
    public TasksIntegrity(CollectionsManagement collectionsManagement) {
        this.collectionsManagement = collectionsManagement;
    }

    /** {@inheritDoc} */
    @Override
    public void doCompute(Context context) throws IOException {
        try {
            removeFinishedEventIfStartIsAbsent(context);
            updateUsageTime(context);
        } catch (ParseException e) {
            throw new IOException(e);
        }
    }

    private void updateUsageTime(Context context) throws ParseException {
        removeObsoleteData(context);

        DBCollection collection = collectionsManagement.getOrCreate(MetricType.TASKS.toString().toLowerCase());

        DBObject q = Utils.setDateFilter(context, STOP_TIME);
        q.put(AbstractMetric.TASK_TYPE, new BasicDBObject("$in", new String[]{"builder", "runner", "debugger"}));
        q.put(AbstractMetric.START_TIME, new BasicDBObject("$exists", true));

        DBCursor cursor = collection.find(q);
        while (cursor.hasNext()) {
            DBObject doc = cursor.next();

            String taskType = doc.get(AbstractMetric.TASK_TYPE).toString();
            long startTime = Long.parseLong(doc.get(AbstractMetric.START_TIME).toString());
            long endTime = Long.parseLong(doc.get(AbstractMetric.STOP_TIME).toString());

            String launchType;
            if (doc.containsField(LAUNCH_TYPE)) {
                launchType = doc.get(LAUNCH_TYPE).toString();
            } else {
                launchType = "timeout";
            }

            long timeout;
{'fraction_non_alphanumeric': 0.09301732925586137, 'fraction_numerical': 0.002420998980632008, 'mean_word_length': 4.18085808580858, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6652055', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2192, 'n_words': 493}