<commit_before>package hu.bme.mit.codemodel.rifle.database;

import java.lang.reflect.Field;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

import com.google.common.base.Stopwatch;
import hu.bme.mit.codemodel.rifle.database.querybuilder.AsgNode;
import hu.bme.mit.codemodel.rifle.database.querybuilder.Query;
import hu.bme.mit.codemodel.rifle.database.querybuilder.QueryBuilder;
import org.neo4j.driver.v1.Transaction;

import com.shapesecurity.functional.Pair;
import com.shapesecurity.functional.data.ConcatList;
import com.shapesecurity.functional.data.Either;
import com.shapesecurity.functional.data.HashTable;
import com.shapesecurity.functional.data.ImmutableList;
import com.shapesecurity.functional.data.Maybe;
import com.shapesecurity.shift.parser.SourceSpan;
import com.shapesecurity.shift.parser.ParserWithLocation;
import com.shapesecurity.shift.scope.Scope;

/**
 * After processing, the parsed AST becomes an ASG.
 */
public class ASTScopeProcessor {
    /**
     * DbServices instance for branch.
     */
    private final DbServices dbServices;

    /**
     * Path of the parsed file.
     */
    private final String parsedFilePath;

    /**
     * A Shape's parser with location.
     */
    private final ParserWithLocation parserWithLocation;

    /**
     * AST items that are already traversed and processed.
     */
    private final Map<Object, Object> processedAstItems = new IdentityHashMap<>();

    /**
     * AST objects with ASG nodes.
     */
    private final Map<Object, AsgNode> objectsWithAsgNodes = new HashMap<>();

    /**
     * The processing queue.
     */
    private final BlockingQueue<QueueItem> processingQueue = new LinkedBlockingQueue<>();

    /**
     * Internal class for processing.
     */
    protected class QueueItem {
        Object subject;
        Object parent;
        String label;

        QueueItem(Object subject, Object parent, String label) {
            this.subject = subject;
            this.parent = parent;
            this.label = label;
        }
    }

    /**
     * Default constructor.
     *
     * @param dbServices
     * @param parsedFilePath
     * @param parserWithLocation
     */
    public ASTScopeProcessor(DbServices dbServices, String parsedFilePath, ParserWithLocation parserWithLocation) {
        this.dbServices = dbServices;
        this.parsedFilePath = parsedFilePath;
        this.parserWithLocation = parserWithLocation;
    }

    /**
     * Processes a given scope with a sessionId.
     * <p>
     * The items (becoming ASG nodes) are stored in a blocking queue.
     *
     * @param scope
     * @param sessionId
     */
    public void processScope(Scope scope, String sessionId) {
        this.createFilePathNode(sessionId);
        processingQueue.add(new QueueItem(scope, null, null));

        final Logger logger = Logger.getLogger("codemodel");

        Stopwatch stopwatch = Stopwatch.createStarted();

        try {
            while (!processingQueue.isEmpty()) {
                QueueItem queueItem = processingQueue.take();
                process(queueItem, sessionId);
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        long scopeDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
        logger.info(String.format("%s %s %dms", parsedFilePath, "SCOPING", scopeDone));

        final List<AsgNode> asgNodes = new ArrayList<>(this.objectsWithAsgNodes.values());

        try (Transaction tx = dbServices.beginTx()) {
            List<Query> queriesToRun = new ArrayList<>();

            stopwatch.reset();
            stopwatch.start();
            queriesToRun.addAll(QueryBuilder.getCreateNodeQueries(asgNodes));
            long assembleDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            logger.info(String.format("%s %s (%s query assembled) %dms", parsedFilePath, "ASSEMBLE", queriesToRun.size(), assembleDone));

            stopwatch.reset();
            stopwatch.start();
            for (Query q : queriesToRun) {
                dbServices.execute(q);
            }
            long createDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            logger.info(String.format("%s %s (%s query executed) %dms", parsedFilePath, "CREATE", queriesToRun.size(), createDone));

            queriesToRun.clear();
            queriesToRun.addAll(QueryBuilder.getSetRelationshipQueries(asgNodes));

            stopwatch.reset();
            stopwatch.start();
            for (Query q : queriesToRun) {
                dbServices.execute(q);
            }
            long relationshipDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
            logger.info(String.format("%s %s (%s query executed) %dms", parsedFilePath, "RELATIONSHIPS", queriesToRun.size(), relationshipDone));

            tx.success();
        } catch (Exception e) {
            e.printStackTrace();
        }
        stopwatch.reset();
    }<commit_msg>Benchmark logging is currently not needed<commit_after>package hu.bme.mit.codemodel.rifle.database;

import java.lang.reflect.Field;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

import com.google.common.base.Stopwatch;
import hu.bme.mit.codemodel.rifle.database.querybuilder.AsgNode;
import hu.bme.mit.codemodel.rifle.database.querybuilder.Query;
import hu.bme.mit.codemodel.rifle.database.querybuilder.QueryBuilder;
import org.neo4j.driver.v1.Transaction;

import com.shapesecurity.functional.Pair;
import com.shapesecurity.functional.data.ConcatList;
import com.shapesecurity.functional.data.Either;
import com.shapesecurity.functional.data.HashTable;
import com.shapesecurity.functional.data.ImmutableList;
import com.shapesecurity.functional.data.Maybe;
import com.shapesecurity.shift.parser.SourceSpan;
import com.shapesecurity.shift.parser.ParserWithLocation;
import com.shapesecurity.shift.scope.Scope;

/**
 * After processing, the parsed AST becomes an ASG.
 */
public class ASTScopeProcessor {
    /**
     * DbServices instance for branch.
     */
    private final DbServices dbServices;

    /**
     * Path of the parsed file.
     */
    private final String parsedFilePath;

    /**
     * A Shape's parser with location.
     */
    private final ParserWithLocation parserWithLocation;

    /**
     * AST items that are already traversed and processed.
     */
    private final Map<Object, Object> processedAstItems = new IdentityHashMap<>();

    /**
     * AST objects with ASG nodes.
     */
    private final Map<Object, AsgNode> objectsWithAsgNodes = new HashMap<>();

    /**
     * The processing queue.
     */
    private final BlockingQueue<QueueItem> processingQueue = new LinkedBlockingQueue<>();

    /**
     * Internal class for processing.
     */
    protected class QueueItem {
        Object subject;
        Object parent;
        String label;

        QueueItem(Object subject, Object parent, String label) {
            this.subject = subject;
            this.parent = parent;
            this.label = label;
        }
    }

    /**
     * Default constructor.
     *
     * @param dbServices
     * @param parsedFilePath
     * @param parserWithLocation
     */
    public ASTScopeProcessor(DbServices dbServices, String parsedFilePath, ParserWithLocation parserWithLocation) {
        this.dbServices = dbServices;
        this.parsedFilePath = parsedFilePath;
        this.parserWithLocation = parserWithLocation;
    }

    /**
     * Processes a given scope with a sessionId.
     * <p>
     * The items (becoming ASG nodes) are stored in a blocking queue.
     *
     * @param scope
     * @param sessionId
     */
    public void processScope(Scope scope, String sessionId) {
        this.createFilePathNode(sessionId);
        processingQueue.add(new QueueItem(scope, null, null));

//        final Logger logger = Logger.getLogger("codemodel");

//        Stopwatch stopwatch = Stopwatch.createStarted();

        try {
            while (!processingQueue.isEmpty()) {
                QueueItem queueItem = processingQueue.take();
                process(queueItem, sessionId);
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
//        long scopeDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
//        logger.info(String.format("%s %s %dms", parsedFilePath, "SCOPING", scopeDone));

        final List<AsgNode> asgNodes = new ArrayList<>(this.objectsWithAsgNodes.values());

        try (Transaction tx = dbServices.beginTx()) {
            List<Query> queriesToRun = new ArrayList<>();

//            stopwatch.reset();
//            stopwatch.start();
            queriesToRun.addAll(QueryBuilder.getCreateNodeQueries(asgNodes));
//            long assembleDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
//            logger.info(String.format("%s %s (%s query assembled) %dms", parsedFilePath, "ASSEMBLE", queriesToRun.size(), assembleDone));

//            stopwatch.reset();
//            stopwatch.start();
            for (Query q : queriesToRun) {
                dbServices.execute(q);
            }
//            long createDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
//            logger.info(String.format("%s %s (%s query executed) %dms", parsedFilePath, "CREATE", queriesToRun.size(), createDone));

            queriesToRun.clear();
            queriesToRun.addAll(QueryBuilder.getSetRelationshipQueries(asgNodes));

//            stopwatch.reset();
//            stopwatch.start();
            for (Query q : queriesToRun) {
                dbServices.execute(q);
            }
//            long relationshipDone = stopwatch.elapsed(TimeUnit.MILLISECONDS);
//            logger.info(String.format("%s %s (%s query executed) %dms", parsedFilePath, "RELATIONSHIPS", queriesToRun.size(), relationshipDone));

            tx.success();
        } catch (Exception e) {
            e.printStackTrace();
        }
//        stopwatch.reset();
    }
{'fraction_non_alphanumeric': 0.10962408974611297, 'fraction_numerical': 0.000393623302499508, 'mean_word_length': 2.655755395683453, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114322', 'n_tokens_mistral': 2877, 'n_tokens_neox': 2786, 'n_words': 605}