<commit_before>/*
 * This program is part of the OpenLMIS logistics management information system platform software.
 * Copyright © 2017 VillageReach
 *
 * This program is free software: you can redistribute it and/or modify it under the terms
 * of the GNU Affero General Public License as published by the Free Software Foundation, either
 * version 3 of the License, or (at your option) any later version.
 *  
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 * See the GNU Affero General Public License for more details. You should have received a copy of
 * the GNU Affero General Public License along with this program. If not, see
 * http://www.gnu.org/licenses.  For additional information contact <EMAIL>. 
 */

package org.openlmis.referencedata;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import org.apache.commons.lang3.tuple.MutablePair;
import org.javers.common.collections.Lists;
import org.openlmis.referencedata.dto.RightAssignmentDto;
import org.openlmis.referencedata.util.Resource2Db;
import org.slf4j.ext.XLogger;
import org.slf4j.ext.XLoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Profile;
import org.springframework.core.annotation.Order;
import org.springframework.core.io.Resource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;

/**
 * RightAssignmentInitializer runs after its associated Spring application has loaded. It 
 * automatically re-generates right assignments into the database, after dropping the existing 
 * right assignments. This component only runs when the "refresh-db" Spring profile is set.
 */
@Component
@Profile("refresh-db")
@Order(10)
public class RightAssignmentInitializer implements CommandLineRunner {

  private static final XLogger XLOGGER = XLoggerFactory.getXLogger(
      RightAssignmentInitializer.class);
  
  private static final String RIGHT_ASSIGNMENTS_PATH = "classpath:db/right-assignments/";

  static final String DELETE_SQL = "DELETE FROM referencedata.right_assignments;";

  @Value(value = RIGHT_ASSIGNMENTS_PATH + "get_right_assignments.sql")
  private Resource rightAssignmentsResource;

  @Value(value = RIGHT_ASSIGNMENTS_PATH + "get_all_supervised_facilities_from_node.sql")
  private Resource supervisedFacilitiesResource;

  @Autowired
  JdbcTemplate template;
  
  /**
   * Re-generates right assignments.
   * @param args command line arguments
   */
  public void run(String... args) throws IOException {
    XLOGGER.entry();
    
    // Drop existing rows; we are regenerating from scratch
    XLOGGER.debug("Drop existing right assignments");
    template.update(DELETE_SQL);

    // Get a right assignment matrix from database
    XLOGGER.debug("Get intermediate right assignments from role assignments");
    List<RightAssignmentDto> dbRightAssignments = getRightAssignmentsFromDbResource(
        rightAssignmentsResource);

    // Convert matrix to a set of right assignments to insert
    XLOGGER.debug("Convert intermediate right assignments to right assignments for insert");
    Set<RightAssignmentDto> rightAssignmentsToInsert = convertForInsert(dbRightAssignments,
        supervisedFacilitiesResource);

    // Convert set of right assignments to insert to a set of SQL inserts
    XLOGGER.debug("Convert right assignments to SQL inserts");
    MutablePair dataWithHeader = new MutablePair<List<String>, List<Object[]>>();
    dataWithHeader.setRight( rightAssignmentsToInsert.stream()
        .map(rad -> rad.toColumnArray())
        .collect(Collectors.toList()) );

    // set column headers
    dataWithHeader.setLeft(Lists.asList("id",
        "userid",
        "rightname",
        "facilityid",
        "programid"));

    // insert into right_assignments
    XLOGGER.debug("Perform SQL inserts");
    Resource2Db r2db = new Resource2Db(template);
    r2db.insertToDbFromBatchedPair("referencedata.right_assignments", dataWithHeader);

    XLOGGER.exit();
  }

  List<RightAssignmentDto> getRightAssignmentsFromDbResource(Resource resource)
      throws IOException {
    return template.query(
        resourceToString(resource),
        (ResultSet rs, int rowNum) -> {

          RightAssignmentDto rightAssignmentMap = new RightAssignmentDto();
          rightAssignmentMap.setUserId(UUID.fromString(rs.getString("userid")));
          rightAssignmentMap.setRightName(rs.getString("rightname"));
          if (null != rs.getString("facilityid")) {
            rightAssignmentMap.setFacilityId(UUID.fromString(rs.getString("facilityid")));
          }
          if (null != rs.getString("programid")) {
            rightAssignmentMap.setProgramId(UUID.fromString(rs.getString("programid")));
          }
          if (null != rs.getString("supervisorynodeid")) {
            rightAssignmentMap.setSupervisoryNodeId(
                UUID.fromString(rs.getString("supervisorynodeid")));
          }
          return rightAssignmentMap;
        }
    );
  }

  Set<RightAssignmentDto> convertForInsert(List<RightAssignmentDto> rightAssignments,
      Resource supervisedFacilitiesResource)
      throws IOException {
    Set<RightAssignmentDto> rightAssignmentsToInsert = new HashSet<>();<commit_msg>OLMIS-3130, partition role assignment processing<commit_after>/*
 * This program is part of the OpenLMIS logistics management information system platform software.
 * Copyright © 2017 VillageReach
 *
 * This program is free software: you can redistribute it and/or modify it under the terms
 * of the GNU Affero General Public License as published by the Free Software Foundation, either
 * version 3 of the License, or (at your option) any later version.
 *  
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
 * See the GNU Affero General Public License for more details. You should have received a copy of
 * the GNU Affero General Public License along with this program. If not, see
 * http://www.gnu.org/licenses.  For additional information contact <EMAIL>. 
 */

package org.openlmis.referencedata;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.sql.ResultSet;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.tuple.MutablePair;
import org.openlmis.referencedata.dto.RightAssignmentDto;
import org.openlmis.referencedata.util.Resource2Db;
import org.slf4j.ext.XLogger;
import org.slf4j.ext.XLoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Profile;
import org.springframework.core.annotation.Order;
import org.springframework.core.io.Resource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.util.StreamUtils;

/**
 * RightAssignmentInitializer runs after its associated Spring application has loaded. It 
 * automatically re-generates right assignments into the database, after dropping the existing 
 * right assignments. This component only runs when the "refresh-db" Spring profile is set.
 */
@Component
@Profile("refresh-db")
@Order(10)
public class RightAssignmentInitializer implements CommandLineRunner {

  private static final XLogger XLOGGER = XLoggerFactory.getXLogger(
      RightAssignmentInitializer.class);
  
  private static final String RIGHT_ASSIGNMENTS_PATH = "classpath:db/right-assignments/";

  static final String DELETE_SQL = "DELETE FROM referencedata.right_assignments;";

  @Value(value = RIGHT_ASSIGNMENTS_PATH + "get_right_assignments.sql")
  private Resource rightAssignmentsResource;

  @Value(value = RIGHT_ASSIGNMENTS_PATH + "get_all_supervised_facilities_from_node.sql")
  private Resource supervisedFacilitiesResource;

  @Autowired
  JdbcTemplate template;
  
  /**
   * Re-generates right assignments.
   * @param args command line arguments
   */
  public void run(String... args) throws IOException {
    XLOGGER.entry();
    
    // Drop existing rows; we are regenerating from scratch
    XLOGGER.debug("Drop existing right assignments");
    template.update(DELETE_SQL);

    // Get a right assignment matrix from database
    XLOGGER.debug("Get intermediate right assignments from role assignments");
    List<RightAssignmentDto> dbRightAssignments = getRightAssignmentsFromDbResource(
        rightAssignmentsResource);

    Resource2Db r2db = new Resource2Db(template);
    for ( List partialRightAssignments : ListUtils.partition(dbRightAssignments, 100) ) {
      insertFromDbRightAssignmentList(r2db, partialRightAssignments);
    }

    XLOGGER.exit();
  }

  private void insertFromDbRightAssignmentList(Resource2Db resource2Db,
                                               List<RightAssignmentDto> rightAssignmentDtos)
      throws IOException {

    // Convert matrix to a set of right assignments to insert
    XLOGGER.debug("Convert intermediate right assignments to right assignments for insert");
    Set<RightAssignmentDto> rightAssignmentsToInsert = convertForInsert(rightAssignmentDtos,
        supervisedFacilitiesResource);

    // Convert set of right assignments to insert to a set of SQL inserts
    XLOGGER.debug("Convert right assignments to SQL inserts");
    MutablePair dataWithHeader = new MutablePair<List<String>, List<Object[]>>();
    dataWithHeader.setRight( rightAssignmentsToInsert.stream()
        .map(rad -> rad.toColumnArray())
        .collect(Collectors.toList()) );

    // set column headers
    dataWithHeader.setLeft(Arrays.asList("id",
        "userid",
        "rightname",
        "facilityid",
        "programid"));

    // insert into right_assignments
    XLOGGER.debug("Perform SQL inserts");
    resource2Db.insertToDbFromBatchedPair("referencedata.right_assignments", dataWithHeader);
  }

  List<RightAssignmentDto> getRightAssignmentsFromDbResource(Resource resource)
      throws IOException {
    return template.query(
        resourceToString(resource),
        (ResultSet rs, int rowNum) -> {

          RightAssignmentDto rightAssignmentMap = new RightAssignmentDto();
          rightAssignmentMap.setUserId(UUID.fromString(rs.getString("userid")));
          rightAssignmentMap.setRightName(rs.getString("rightname"));
          if (null != rs.getString("facilityid")) {
            rightAssignmentMap.setFacilityId(UUID.fromString(rs.getString("facilityid")));
          }
          if (null != rs.getString("programid")) {
            rightAssignmentMap.setProgramId(UUID.fromString(rs.getString("programid")));
          }
          if (null != rs.getString("supervisorynodeid")) {
            rightAssignmentMap.setSupervisoryNodeId(
                UUID.fromString(rs.getString("supervisorynodeid")));
          }
          return rightAssignmentMap;
        }
    );
  }

  Set<RightAssignmentDto> convertForInsert(List<RightAssignmentDto> rightAssignments,
      Resource supervisedFacilitiesResource)
      throws IOException {
    Set<RightAssignmentDto> rightAssignmentsToInsert = new HashSet<>();
{'fraction_non_alphanumeric': 0.0859591630955382, 'fraction_numerical': 0.003445088648012772, 'mean_word_length': 4.179286335944299, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5397795', 'n_tokens_mistral': 3152, 'n_tokens_neox': 3154, 'n_words': 850}