<commit_before>import {ReportJsonRepository} from '../repositories/json/report-json.repo';
import {ProjectSettingsJsonRepository} from "../repositories/json/project-settings-json.repo";
import {ReportRepository} from '../repositories/report.repo';
import {ProjectSettingsRepository} from "../repositories/project-settings.repo";
import {
  Project,
  TreeNode,
  Dependency,
  Path,
  Problem,
  ProjectInfo
} from "microdocs-core-ts/dist/domain";
import {NOTICE, ERROR} from "microdocs-core-ts/dist/domain/problem/problem-level.model";
import {REST} from "microdocs-core-ts/dist/domain/dependency/dependency-type.model";

import {ProblemReporter, SchemaHelper} from "microdocs-core-ts/dist/helpers";
import {getProblemsInProject, getProblemsInDependency} from "microdocs-core-ts/dist/helpers";

import {PathCheck} from "../checks/path-check";
import {QueryParamsCheck} from "../checks/query-params.check";
import {BodyParamsCheck} from "../checks/body-params.check";
import {PathParamsCheck} from "../checks/path-params.check";
import {ResponseCheck} from "../checks/response.check";
import {ProjectService} from "./project.service";

export class AggregationService {

  private endpointChecks:PathCheck[] = [new QueryParamsCheck(), new BodyParamsCheck(), new PathParamsCheck(), new ResponseCheck()];
  private reportRepo:ReportRepository;
  private projectSettingsRepo:ProjectSettingsRepository;
  private projectService:ProjectService;

  constructor() {
    this.reportRepo = ReportJsonRepository.bootstrap();
    this.projectSettingsRepo = ProjectSettingsJsonRepository.bootstrap();
    this.projectService = ProjectService.bootstrap();
  }

  public static bootstrap():AggregationService {
    return new AggregationService();
  }

  /**
   * Check new project for breaking changes
   * @param project
   * @returns {Problem[]}
   */
  public checkProject(env:string, project:Project):Problem[] {
    // Fix project
    this.fixDependencyUpperCase(project);

    // Load all projects
    var projectCache = this.loadProjects(env);

    // check dependencies
    var node = new TreeNode();
    this.resolveDependencies(env, project, projectCache, node);

    // check other projects for breaking changes
    var clientProblems = this.reverseCheckDependencies(env, project, projectCache, node);

    // collect problems
    var problems = getProblemsInProject(project);
    clientProblems.forEach(problem => problems.push(problem));

    return problems;
  }

  /**
   * Start the reindex process
   * @return {TreeNode}
   */
  public reindex(env:string):TreeNode {
    console.info("Start reindex");

    // Load all projects
    var projectCache = this.loadProjects(env);

    console.info("Build dependency tree");
    var tree = this.buildDependencyTree(env, projectCache);

    console.info("Store aggregations");
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        var project = projectCache[title][version];
        this.projectService.storeAggregatedProject(env, project);
      }
    }
    this.projectService.storeAggregatedProjects(env, tree);

    console.info("Finish reindex");

    return tree;
  }

  /**
   * Build dependency tree
   * @param projects list of all projects
   * @return {TreeNode} result
   */
  public buildDependencyTree(env:string, projectCache:{[title:string]:{[version:string]:Project}}):TreeNode {
    // create rootnode
    var rootNode = new TreeNode();

    // add of all projects the latest version
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        var project = projectCache[title][version];
        var node = new TreeNode();
        node.parent = rootNode;
        node.group = project.info.group;
        node.version = project.info.version;
        node.versions = project.info.versions;
        rootNode.dependencies[project.info.title] = node;
      }
    }

    var copyProjectCache:{[title:string]:{[version:string]:Project}} = {};
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        if (copyProjectCache[title] == undefined) {
          copyProjectCache[title] = {};
        }
        copyProjectCache[title][version] = projectCache[title][version];
      }
    }

    // resolve dependencies
    for (var title in copyProjectCache) {
      for (var version in copyProjectCache[title]) {
        var project = copyProjectCache[title][version];
        var aggregatedProject = this.resolveDependencies(env, project, projectCache, rootNode.dependencies[project.info.title]);
      }
    }

    return rootNode;
  }

  /**
   * Fix uppercase dependency names
   * @param project
   */
  private fixDependencyUpperCase(project:Project){
    if(project.dependencies){
      var fixedDependencies = {};
      for(var name in project.dependencies){
        fixedDependencies[name.toLowerCase()] = project.dependencies[name];
      }
      project.dependencies = fixedDependencies;
    }
  }

  /**
   * Load all projects
   * @return all project structured as [name].[version].[project]
   */
  private loadProjects(env:string):{[title:string]:{[version:string]:Project}} {
    var projectCache:{[title:string]:{[version:string]:Project}} = {};
    // var projects:Project[] = [];
    var projectInfos = this.reportRepo.getProjects(env);
    projectInfos.forEach(projectInfo => {
      try {
        var project = this.reportRepo.getProject(env, projectInfo);
        if (project != null) {
          this.fixDependencyUpperCase(project);
          project = this.mergeProjectSettings(project);
          if (projectCache[project.info.title] == null || projectCache[project.info.title] == undefined) {
            projectCache[project.info.title] = {};
          }
          projectCache[project.info.title][project.info.version] = project;
        }
      } catch (e) {
        console.error("Failed to load project: " + projectInfo.title);
        console.error(e);
      }
    });
    return projectCache;
  }

  /**
   *
   * @param project
   * @param projectCache
   * @param parentNode
   */
  private reverseCheckDependencies(env:string, project:Project, projectCache:{[title:string]:{[version:string]:Project}}, parentNode:TreeNode):Problem[] {
    var problems:Problem[] = [];
    for (var title in projectCache) {
      var versions = Object.keys(projectCache[title]).sort();
      if(versions.length > 0){
        var latestVersion = versions[versions.length-1];
        var clientProject = projectCache[title][latestVersion];
        if (clientProject.dependencies != null && clientProject.definitions != undefined) {
          if (clientProject.dependencies[project.info.title] != undefined) {
            var dependency = clientProject.dependencies[project.info.title];
            this.resolveDependency(env, clientProject, project.info.title, dependency, projectCache, parentNode);

            // map problems to the produces endpoints
            var problemReport = new ProblemReporter(project);
            if(dependency.paths != undefined && dependency.paths != null){
              for(var path in dependency.paths){
                for( var method in dependency.paths[path]){
                  var endpoint = dependency.paths[path][method];
                  if(endpoint.problems != undefined && endpoint.problems != null){
                    var producerEndpoint = SchemaHelper.resolveReference('paths.' + path + "." + method, project);
                    endpoint.problems.forEach(problem => {
                      problemReport.report(
                        problem.level,
                        problem.message,
                        producerEndpoint ? producerEndpoint.controller : undefined,
                        producerEndpoint ? producerEndpoint.method : undefined,
                        clientProject,
                        title,
                        latestVersion,
                        endpoint ? endpoint.controller : undefined,<commit_msg>MXTS-242: Fix problems checking<commit_after>import {ReportJsonRepository} from '../repositories/json/report-json.repo';
import {ProjectSettingsJsonRepository} from "../repositories/json/project-settings-json.repo";
import {ReportRepository} from '../repositories/report.repo';
import {ProjectSettingsRepository} from "../repositories/project-settings.repo";
import {
  Project,
  TreeNode,
  Dependency,
  Path,
  Problem,
  ProjectInfo
} from "microdocs-core-ts/dist/domain";
import {NOTICE, ERROR} from "microdocs-core-ts/dist/domain/problem/problem-level.model";
import {REST} from "microdocs-core-ts/dist/domain/dependency/dependency-type.model";

import {ProblemReporter, SchemaHelper} from "microdocs-core-ts/dist/helpers";
import {getProblemsInProject, getProblemsInDependency} from "microdocs-core-ts/dist/helpers";

import {PathCheck} from "../checks/path-check";
import {QueryParamsCheck} from "../checks/query-params.check";
import {BodyParamsCheck} from "../checks/body-params.check";
import {PathParamsCheck} from "../checks/path-params.check";
import {ResponseCheck} from "../checks/response.check";
import {ProjectService} from "./project.service";

export class AggregationService {

  private endpointChecks:PathCheck[] = [new QueryParamsCheck(), new BodyParamsCheck(), new PathParamsCheck(), new ResponseCheck()];
  private reportRepo:ReportRepository;
  private projectSettingsRepo:ProjectSettingsRepository;
  private projectService:ProjectService;

  constructor() {
    this.reportRepo = ReportJsonRepository.bootstrap();
    this.projectSettingsRepo = ProjectSettingsJsonRepository.bootstrap();
    this.projectService = ProjectService.bootstrap();
  }

  public static bootstrap():AggregationService {
    return new AggregationService();
  }

  /**
   * Check new project for breaking changes
   * @param project
   * @returns {Problem[]}
   */
  public checkProject(env:string, project:Project):Problem[] {
    // Fix project
    this.fixDependencyUpperCase(project);

    // Load all projects
    var projectCache = this.loadProjects(env);

    // check dependencies
    var node = new TreeNode();
    this.resolveDependencies(env, project, projectCache, node);

    // check other projects for breaking changes
    var clientProblems = this.reverseCheckDependencies(env, project, projectCache, node);

    // collect problems
    var problems = getProblemsInProject(project);
    clientProblems.forEach(problem => problems.push(problem));

    return problems;
  }

  /**
   * Start the reindex process
   * @return {TreeNode}
   */
  public reindex(env:string):TreeNode {
    console.info("Start reindex");

    // Load all projects
    var projectCache = this.loadProjects(env);

    console.info("Build dependency tree");
    var tree = this.buildDependencyTree(env, projectCache);

    console.info("Store aggregations");
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        var project = projectCache[title][version];
        this.projectService.storeAggregatedProject(env, project);
      }
    }
    this.projectService.storeAggregatedProjects(env, tree);

    console.info("Finish reindex");

    return tree;
  }

  /**
   * Build dependency tree
   * @param projects list of all projects
   * @return {TreeNode} result
   */
  public buildDependencyTree(env:string, projectCache:{[title:string]:{[version:string]:Project}}):TreeNode {
    // create rootnode
    var rootNode = new TreeNode();

    // add of all projects the latest version
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        var project = projectCache[title][version];
        var node = new TreeNode();
        node.parent = rootNode;
        node.group = project.info.group;
        node.version = project.info.version;
        node.versions = project.info.versions;
        rootNode.dependencies[project.info.title] = node;
      }
    }

    var copyProjectCache:{[title:string]:{[version:string]:Project}} = {};
    for (var title in projectCache) {
      for (var version in projectCache[title]) {
        if (copyProjectCache[title] == undefined) {
          copyProjectCache[title] = {};
        }
        copyProjectCache[title][version] = projectCache[title][version];
      }
    }

    // resolve dependencies
    for (var title in copyProjectCache) {
      for (var version in copyProjectCache[title]) {
        var project = copyProjectCache[title][version];
        var aggregatedProject = this.resolveDependencies(env, project, projectCache, rootNode.dependencies[project.info.title]);
      }
    }

    return rootNode;
  }

  /**
   * Fix uppercase dependency names
   * @param project
   */
  private fixDependencyUpperCase(project:Project){
    if(project.dependencies){
      var fixedDependencies = {};
      for(var name in project.dependencies){
        fixedDependencies[name.toLowerCase()] = project.dependencies[name];
      }
      project.dependencies = fixedDependencies;
    }
  }

  /**
   * Load all projects
   * @return all project structured as [name].[version].[project]
   */
  private loadProjects(env:string):{[title:string]:{[version:string]:Project}} {
    var projectCache:{[title:string]:{[version:string]:Project}} = {};
    // var projects:Project[] = [];
    var projectInfos = this.reportRepo.getProjects(env);
    projectInfos.forEach(projectInfo => {
      try {
        var project = this.reportRepo.getProject(env, projectInfo);
        if (project != null) {
          this.fixDependencyUpperCase(project);
          project = this.mergeProjectSettings(project);
          if (projectCache[project.info.title] == null || projectCache[project.info.title] == undefined) {
            projectCache[project.info.title] = {};
          }
          projectCache[project.info.title][project.info.version] = project;
        }
      } catch (e) {
        console.error("Failed to load project: " + projectInfo.title);
        console.error(e);
      }
    });
    return projectCache;
  }

  /**
   *
   * @param project
   * @param projectCache
   * @param parentNode
   */
  private reverseCheckDependencies(env:string, project:Project, projectCache:{[title:string]:{[version:string]:Project}}, parentNode:TreeNode):Problem[] {
    var problems:Problem[] = [];
    if(!projectCache[project.info.title]) {
      projectCache[project.info.title] = {};
    }
    projectCache[project.info.title][project.info.version] = project;
    for (var title in projectCache) {
      var versions = Object.keys(projectCache[title]).sort();
      if(versions.length > 0){
        var latestVersion = versions[versions.length-1];
        var clientProject = projectCache[title][latestVersion];
        if (clientProject.dependencies != null && clientProject.definitions != undefined) {
          if (clientProject.dependencies[project.info.title] != undefined) {
            var dependency = clientProject.dependencies[project.info.title];
            this.resolveDependency(env, clientProject, project.info.title, dependency, projectCache, parentNode);

            // map problems to the produces endpoints
            var problemReport = new ProblemReporter(project);
            if(dependency.paths != undefined && dependency.paths != null){
              for(var path in dependency.paths){
                for( var method in dependency.paths[path]){
                  var endpoint = dependency.paths[path][method];
                  if(endpoint.problems != undefined && endpoint.problems != null){
                    var producerEndpoint = SchemaHelper.resolveReference('paths.' + path + "." + method, project);
                    endpoint.problems.forEach(problem => {
                      problemReport.report(
                        problem.level,
                        problem.message,
                        producerEndpoint ? producerEndpoint.controller : undefined,
                        producerEndpoint ? producerEndpoint.method : undefined,
                        clientProject,
                        title,
                        latestVersion,
                        endpoint ? endpoint.controller : undefined,
{'fraction_non_alphanumeric': 0.11873645091359554, 'fraction_numerical': 0.0004335707649427067, 'mean_word_length': 3.095890410958904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7584685', 'n_tokens_mistral': 4320, 'n_tokens_neox': 4131, 'n_words': 926}