<commit_before>library crossdart.store;

import 'dart:async';
import 'package:crossdart/src/db_pool.dart';
import 'package:crossdart/src/parsed_data.dart';
import 'package:crossdart/src/environment.dart';
import 'package:crossdart/src/entity.dart';
import 'package:crossdart/src/package.dart';
import 'package:crossdart/src/package_info.dart';
import 'package:crossdart/src/location.dart';
import 'package:crossdart/src/config.dart';
import 'package:sqljocky/sqljocky.dart';
import 'package:logging/logging.dart';

var _logger = new Logger("store");

Map<Type, int> entityTypeIds = {
  Reference: 1,
  Declaration: 2,
  Import: 3,
  Token: 4
};

Map<EntityKind, int> entityKindIds = {
  EntityKind.CLASS: 1,
  EntityKind.METHOD: 2,
  EntityKind.LOCAL_VARIABLE: 3,
  EntityKind.FUNCTION: 4,
  EntityKind.PROPERTY_ACCESSOR: 5,
  EntityKind.CONSTRUCTOR: 6,
  EntityKind.FIELD: 7,
  EntityKind.FUNCTION_TYPE_ALIAS: 8,
  EntityKind.TOP_LEVEL_VARIABLE: 9
};

// TODO: Refactor to a class

Future store(Environment environment, ParsedData parsedData) async {
  _logger.info("Starting transaction to store the package, dependencies, and all the entities...");

  var config = environment.config;
  await dbPool(environment.config).query("SET SESSION innodb_lock_wait_timeout=2000");
  final transaction = await dbPool(environment.config).startTransaction(consistent: false);
  await transaction.query("SET SESSION innodb_lock_wait_timeout=2000");

  for (final package in environment.packages) {
    await storePackage(config, package.packageInfo, package.source, package.description, transaction);
  }
  final environmentWithIds = await environment.rebuildWithPackageIds(transaction);
  await storeDependencies(environmentWithIds, environmentWithIds.package, transaction);

  final insertQuery = await transaction.prepare("""
    INSERT IGNORE INTO entities (declaration_id, type, kind, name, context_name, offset, end, line_number, line_offset, path, package_id, created_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  """);
  var files = [];

  _logger.info("Preparing files");

  final packages = parsedData.files.keys.map((k) => environmentWithIds.packagesByFiles[k]).toSet();

  final query = """
    SELECT DISTINCT package_id FROM entities WHERE package_id IN (${packages.map((p) => p.id).toList().join(",")})
  """;
  var existingPackageIds = (await (await transaction.query(query)).toList()).map((r) => r.package_id).toSet();

  parsedData.files.forEach((absolutePath, entities) {
    var location = new Location.fromEnvironment(environmentWithIds, absolutePath);
    if (!existingPackageIds.contains(location.package.id)) {
      files.add([absolutePath, entities]);
    }
  });

  _logger.info("Storing declarations");
  var idsByDeclarationsList = [];
  for (final absolutePath in parsedData.files.keys) {
    var entities = parsedData.files[absolutePath];
    var filteredEntities = entities.where((e) => e is Declaration);
    idsByDeclarationsList.add(await _storeDeclarations(environmentWithIds, insertQuery, absolutePath, filteredEntities, transaction));
  }

  _logger.info("Making idsByDeclaration map");
  var idsByDeclarations = idsByDeclarationsList.fold({}, (memo, map) {
    memo.addAll(map);
    return memo;
  });

  _logger.info("Storing references");
  var referencesValues = files.map((tuple) {
    var absolutePath = tuple[0];
    var entities = tuple[1];
    var filteredEntities = entities.where((e) => e is Reference);

    return _getReferencesValues(environmentWithIds, parsedData, absolutePath, filteredEntities, idsByDeclarations);
  }).expand((i) => i).toList();
  _logger.info("Executing query to store references");
  if (referencesValues.length > 0) {
    await insertQuery.executeMulti(referencesValues);
  }

  _logger.info("Storing tokens");
  var tokensValues = files.map((tuple) {
    var absolutePath = tuple[0];
    var entities = tuple[1];
    var filteredEntities = entities.where((e) => e.runtimeType == Token);
    return _getTokensValues(environmentWithIds, parsedData, absolutePath, filteredEntities);
  }).expand((i) => i).toList();
  _logger.info("Executing query to store tokens");
  if (tokensValues.length > 0) {
    await insertQuery.executeMulti(tokensValues);
  }
  _logger.info("Committing transaction");
  transaction.commit();
}

List _buildValue(Config config, Entity entity, Location location, [int declarationId]) {
  return [
      declarationId,
      entityTypeIds[entity.runtimeType],
      entityKindIds[entity.kind],
      entity.name,
      entity.contextName,
      entity.offset,
      entity.end,
      entity.lineNumber,
      entity.lineOffset,
      location.path,
      location.package.id,
      config.currentDate];
}

Future<Map<Declaration, int>> _storeDeclarations(Environment environment, Query query, String absolutePath, Iterable<Declaration> declarations, QueriableConnection conn) async {
  var location = new Location.fromEnvironment(environment, absolutePath);
  var idAndDeclaration = [];
  for (final declaration in declarations) {
    int id;
    if (declaration.id == null) {
      var value = _buildValue(environment.config, declaration, location);
      Results result = await query.execute(value);
      id = result.insertId;
      if (id == 0) {
        id = declaration.id;
      }
      if (id == null) {
        id = (await (await conn.query("""
          SELECT id FROM entities
          WHERE type = ${entityTypeIds[Declaration]} AND offset = ${declaration.offset} AND end = ${declaration.end} AND
                path = '${location.path}' AND package_id = ${location.package.id}
        """)).toList()).first.id;
      }
    } else {<commit_msg>Add a bit more logging<commit_after>library crossdart.store;

import 'dart:io';
import 'dart:async';
import 'package:crossdart/src/db_pool.dart';
import 'package:crossdart/src/parsed_data.dart';
import 'package:crossdart/src/environment.dart';
import 'package:crossdart/src/entity.dart';
import 'package:crossdart/src/package.dart';
import 'package:crossdart/src/package_info.dart';
import 'package:crossdart/src/location.dart';
import 'package:crossdart/src/config.dart';
import 'package:sqljocky/sqljocky.dart';
import 'package:logging/logging.dart';

var _logger = new Logger("store");

Map<Type, int> entityTypeIds = {
  Reference: 1,
  Declaration: 2,
  Import: 3,
  Token: 4
};

Map<EntityKind, int> entityKindIds = {
  EntityKind.CLASS: 1,
  EntityKind.METHOD: 2,
  EntityKind.LOCAL_VARIABLE: 3,
  EntityKind.FUNCTION: 4,
  EntityKind.PROPERTY_ACCESSOR: 5,
  EntityKind.CONSTRUCTOR: 6,
  EntityKind.FIELD: 7,
  EntityKind.FUNCTION_TYPE_ALIAS: 8,
  EntityKind.TOP_LEVEL_VARIABLE: 9
};

// TODO: Refactor to a class

Future store(Environment environment, ParsedData parsedData) async {
  _logger.info("Starting transaction to store the package, dependencies, and all the entities...");

  var config = environment.config;
  await dbPool(environment.config).query("SET SESSION innodb_lock_wait_timeout=2000");
  final transaction = await dbPool(environment.config).startTransaction(consistent: false);
  await transaction.query("SET SESSION innodb_lock_wait_timeout=2000");

  for (final package in environment.packages) {
    await storePackage(config, package.packageInfo, package.source, package.description, transaction);
  }
  final environmentWithIds = await environment.rebuildWithPackageIds(transaction);
  await storeDependencies(environmentWithIds, environmentWithIds.package, transaction);

  final insertQuery = await transaction.prepare("""
    INSERT IGNORE INTO entities (declaration_id, type, kind, name, context_name, offset, end, line_number, line_offset, path, package_id, created_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  """);
  var files = [];

  _logger.info("Preparing files");

  final packages = parsedData.files.keys.map((k) => environmentWithIds.packagesByFiles[k]).toSet();

  final query = """
    SELECT DISTINCT package_id FROM entities WHERE package_id IN (${packages.map((p) => p.id).toList().join(",")})
  """;
  var existingPackageIds = (await (await transaction.query(query)).toList()).map((r) => r.package_id).toSet();

  parsedData.files.forEach((absolutePath, entities) {
    var location = new Location.fromEnvironment(environmentWithIds, absolutePath);
    if (!existingPackageIds.contains(location.package.id)) {
      files.add([absolutePath, entities]);
    }
  });

  _logger.info("Storing declarations");
  var idsByDeclarationsList = [];
  for (final absolutePath in parsedData.files.keys) {
    var entities = parsedData.files[absolutePath];
    var filteredEntities = entities.where((e) => e is Declaration);
    idsByDeclarationsList.add(await _storeDeclarations(environmentWithIds, insertQuery, absolutePath, filteredEntities, transaction));
  }

  _logger.info("\nMaking idsByDeclaration map");
  var idsByDeclarations = idsByDeclarationsList.fold({}, (memo, map) {
    memo.addAll(map);
    return memo;
  });
  _logger.info("Number of declarations - ${idsByDeclarations.length}");

  _logger.info("Storing references");
  var referencesValues = files.map((tuple) {
    var absolutePath = tuple[0];
    var entities = tuple[1];
    var filteredEntities = entities.where((e) => e is Reference);

    return _getReferencesValues(environmentWithIds, parsedData, absolutePath, filteredEntities, idsByDeclarations);
  }).expand((i) => i).toList();
  _logger.info("Executing query to store references - ${referencesValues.length}");
  if (referencesValues.length > 0) {
    await insertQuery.executeMulti(referencesValues);
  }

  _logger.info("Storing tokens");
  var tokensValues = files.map((tuple) {
    var absolutePath = tuple[0];
    var entities = tuple[1];
    var filteredEntities = entities.where((e) => e.runtimeType == Token);
    return _getTokensValues(environmentWithIds, parsedData, absolutePath, filteredEntities);
  }).expand((i) => i).toList();
  _logger.info("Executing query to store tokens - ${tokensValues.length}");
  if (tokensValues.length > 0) {
    await insertQuery.executeMulti(tokensValues);
  }
  _logger.info("Committing transaction");
  transaction.commit();
}

List _buildValue(Config config, Entity entity, Location location, [int declarationId]) {
  return [
      declarationId,
      entityTypeIds[entity.runtimeType],
      entityKindIds[entity.kind],
      entity.name,
      entity.contextName,
      entity.offset,
      entity.end,
      entity.lineNumber,
      entity.lineOffset,
      location.path,
      location.package.id,
      config.currentDate];
}

Future<Map<Declaration, int>> _storeDeclarations(Environment environment, Query query, String absolutePath, Iterable<Declaration> declarations, QueriableConnection conn) async {
  var location = new Location.fromEnvironment(environment, absolutePath);
  var idAndDeclaration = [];
  for (final declaration in declarations) {
    int id;
    if (declaration.id == null) {
      var value = _buildValue(environment.config, declaration, location);
      Results result = await query.execute(value);
      stdout.write(".");
      id = result.insertId;
      if (id == 0) {
        id = declaration.id;
      }
      if (id == null) {
        id = (await (await conn.query("""
          SELECT id FROM entities
          WHERE type = ${entityTypeIds[Declaration]} AND offset = ${declaration.offset} AND end = ${declaration.end} AND
                path = '${location.path}' AND package_id = ${location.package.id}
        """)).toList()).first.id;
      }
    } else {
{'fraction_non_alphanumeric': 0.12928782866097857, 'fraction_numerical': 0.004875500609437576, 'mean_word_length': 4.498803255146003, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7417058', 'n_tokens_mistral': 3491, 'n_tokens_neox': 3424, 'n_words': 767}