<commit_before>package fi.otavanopisto.kuntaapi.server.integrations.ptv.updaters;

import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.ejb.AccessTimeout;
import javax.ejb.Singleton;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.event.Event;
import javax.inject.Inject;

import fi.metatavu.kuntaapi.server.rest.model.Code;
import fi.metatavu.kuntaapi.server.rest.model.LocalizedValue;
import fi.metatavu.ptv.client.ApiResponse;
import fi.metatavu.ptv.client.CodeListApi;
import fi.metatavu.ptv.client.model.VmOpenApiCodeListItem;
import fi.metatavu.ptv.client.model.VmOpenApiCodeListPage;
import fi.metatavu.ptv.client.model.VmOpenApiDialCodeListItem;
import fi.otavanopisto.kuntaapi.server.cache.ModificationHashCache;
import fi.otavanopisto.kuntaapi.server.controllers.IdentifierController;
import fi.otavanopisto.kuntaapi.server.discover.EntityUpdater;
import fi.otavanopisto.kuntaapi.server.id.CodeId;
import fi.otavanopisto.kuntaapi.server.index.IndexRequest;
import fi.otavanopisto.kuntaapi.server.index.IndexableCode;
import fi.otavanopisto.kuntaapi.server.integrations.KuntaApiIdFactory;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.CodeType;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.PtvIdFactory;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.client.PtvApi;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeListTask;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeListTaskQueue;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeResourceContainer;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.translation.PtvTranslator;
import fi.otavanopisto.kuntaapi.server.persistence.model.Identifier;

@ApplicationScoped
@Singleton
@AccessTimeout (unit = TimeUnit.HOURS, value = 1l)
@SuppressWarnings ("squid:S3306")
public class PtvCodeListEntityUpdater extends EntityUpdater {

  @Inject
  private Logger logger;
  
  @Inject
  private PtvCodeResourceContainer ptvCodeResourceContainer;

  @Inject
  private PtvApi ptvApi;
  
  @Inject
  private PtvTranslator ptvTranslator;
  
  @Inject
  private KuntaApiIdFactory kuntaApiIdFactory;

  @Inject
  private PtvIdFactory ptvIdFactory;

  @Inject
  private IdentifierController identifierController;

  @Inject
  private Event<IndexRequest> indexRequest;
  
  @Inject
  private PtvCodeListTaskQueue codeListTaskQueue;

  @Inject
  private ModificationHashCache modificationHashCache;

  @Override
  public String getName() {
    return "ptv-codes";
  }

  @Override
  public void timeout() {
    executeNextTask();
  }
  
  private void executeNextTask() {
    PtvCodeListTask task = codeListTaskQueue.next();
    if (task != null) {
      executeTask(task);
    } else if (codeListTaskQueue.isEmptyAndLocalNodeResponsible()) {
      fillQueue();
    }
  }
  
  private void executeTask(PtvCodeListTask task) {
    CodeListApi codeListApi = ptvApi.getCodeListApi();
    
    switch (task.getType()) {
      case BUSINESS_REGIONS:
        executeAreaCodeListTask(codeListApi, "BusinessRegions", task.getType());
      break;
      case COUNTRY:
        handleCountryCodeListResponse(codeListApi.apiV7CodeListGetCountryCodesGet(), task.getType());
      break;
      case HOSPITAL_REGIONS:
        executeAreaCodeListTask(codeListApi, "HospitalRegions", task.getType());
      break;
      case LANGUAGE:
        handleCodeListResponse(codeListApi.apiV7CodeListGetLanguageCodesGet(), task.getType());
      break;
      case MUNICIPALITY:
        handleCodeListResponse(codeListApi.apiV7CodeListGetMunicipalityCodesGet(), task.getType());
      break;
      case POSTAL:
        handleCodeListPagedResponse(codeListApi.apiV7CodeListGetPostalCodesGet(task.getPage()), task.getType());
      break;
      case PROVINCE:
        executeAreaCodeListTask(codeListApi, "Province", task.getType());
      break;
    }
    
  }

  private void executeAreaCodeListTask(CodeListApi codeListApi, String areaCodeType, CodeType codeType) {
    handleCodeListResponse(codeListApi.apiV7CodeListGetAreaCodesTypeByTypeGet(areaCodeType), codeType);
  }

  private void handleCodeListResponse(ApiResponse<List<VmOpenApiCodeListItem>> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    
    if (response.isOk()) {
      for (VmOpenApiCodeListItem item : response.getResponse()) {
        handleItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }


  private void handleCodeListPagedResponse(ApiResponse<VmOpenApiCodeListPage> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    
    if (response.isOk()) {
      for (VmOpenApiCodeListItem item : response.getResponse().getItemList()) {
        handleItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update paged code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }

  private void handleCountryCodeListResponse(ApiResponse<List<VmOpenApiDialCodeListItem>> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    if (response.isOk()) {
      for (VmOpenApiDialCodeListItem item : response.getResponse()) {
        handleDialCodeItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update country code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }
  
  private void handleDialCodeItem(Long orderIndex, VmOpenApiDialCodeListItem ptvItem, CodeType codeType) {
    CodeId ptvCodeId = ptvIdFactory.createCodeId(String.format("%s-%s", codeType, ptvItem.getCode()));
    Identifier identifier = identifierController.acquireIdentifier(orderIndex, ptvCodeId);
    CodeId kuntaApiCodeId = kuntaApiIdFactory.createFromIdentifier(CodeId.class, identifier);
    updateItem(orderIndex, kuntaApiCodeId, ptvTranslator.translateCode(kuntaApiCodeId, ptvItem, codeType));
  }

  private void handleItem(Long orderIndex, VmOpenApiCodeListItem ptvItem, CodeType codeType) {
    CodeId ptvCodeId = ptvIdFactory.createCodeId(String.format("%s-%s", codeType, ptvItem.getCode()));
    Identifier identifier = identifierController.acquireIdentifier(orderIndex, ptvCodeId);
    CodeId kuntaApiCodeId = kuntaApiIdFactory.createFromIdentifier(CodeId.class, identifier);
    updateItem(orderIndex, kuntaApiCodeId, ptvTranslator.translateCode(kuntaApiCodeId, ptvItem, codeType));
  }
  
  private void updateItem(Long orderIndex, CodeId kuntaApiCodeId, Code code) {
    if (code != null) {
      modificationHashCache.put(kuntaApiCodeId.getId(), createPojoHash(code));
      ptvCodeResourceContainer.put(kuntaApiCodeId, code);
      index(kuntaApiCodeId.getId(), code, orderIndex);
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to translate code %s", kuntaApiCodeId));
    }
  }
  
  private void index(String kuntaApiId, Code code, Long orderIndex) {
    String nameFi = null;
    String nameSv = null;
    String nameEn = null;
    
    for (LocalizedValue localizedValue : code.getNames()) {
      switch (localizedValue.getLanguage()) {
        case "fi":
          nameFi = localizedValue.getValue();
        break;
        case "sv":
          nameSv = localizedValue.getValue();
        break;
        case "en":
          nameEn = localizedValue.getValue();
        break;
      }
    }
    
    IndexableCode indexableCode = new IndexableCode(orderIndex, kuntaApiId, code.getType(), code.getCode(), nameFi, nameSv, nameEn);
    indexRequest.fire(new IndexRequest(indexableCode));  
  }

  private void fillQueue() {
    CodeListApi codeListApi = ptvApi.getCodeListApi();
    
    for (CodeType type : CodeType.values()) {
      if (type == CodeType.POSTAL) {
        fillQueuePostal(codeListApi);
      } else {
        codeListTaskQueue.enqueueTask(false, new PtvCodeListTask(type, 0));
      }
    }
  }

  private void fillQueuePostal(CodeListApi codeListApi) {
    ApiResponse<VmOpenApiCodeListPage> postalCodesResponse = codeListApi.apiV7CodeListGetPostalCodesGet(0);
    if (postalCodesResponse.isOk()) {
      if (postalCodesResponse.getResponse() == null) {
        logger.log(Level.SEVERE, "Failed to read PTV code list size");
      } else {
        for (int page = 0; page < postalCodesResponse.getResponse().getPageCount(); page++) {
          codeListTaskQueue.enqueueTask(false, new PtvCodeListTask(CodeType.POSTAL, page));
        }
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to read PTV code list size [%d]: %s", postalCodesResponse.getStatus(), postalCodesResponse.getMessage()));
    }
  }
  
}
<commit_msg>reduced code list interval on test mode<commit_after>package fi.otavanopisto.kuntaapi.server.integrations.ptv.updaters;

import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.ejb.AccessTimeout;
import javax.ejb.Singleton;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.event.Event;
import javax.inject.Inject;

import fi.metatavu.kuntaapi.server.rest.model.Code;
import fi.metatavu.kuntaapi.server.rest.model.LocalizedValue;
import fi.metatavu.ptv.client.ApiResponse;
import fi.metatavu.ptv.client.CodeListApi;
import fi.metatavu.ptv.client.model.VmOpenApiCodeListItem;
import fi.metatavu.ptv.client.model.VmOpenApiCodeListPage;
import fi.metatavu.ptv.client.model.VmOpenApiDialCodeListItem;
import fi.otavanopisto.kuntaapi.server.cache.ModificationHashCache;
import fi.otavanopisto.kuntaapi.server.controllers.IdentifierController;
import fi.otavanopisto.kuntaapi.server.discover.EntityUpdater;
import fi.otavanopisto.kuntaapi.server.id.CodeId;
import fi.otavanopisto.kuntaapi.server.index.IndexRequest;
import fi.otavanopisto.kuntaapi.server.index.IndexableCode;
import fi.otavanopisto.kuntaapi.server.integrations.KuntaApiIdFactory;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.CodeType;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.PtvIdFactory;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.client.PtvApi;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeListTask;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeListTaskQueue;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.resources.PtvCodeResourceContainer;
import fi.otavanopisto.kuntaapi.server.integrations.ptv.translation.PtvTranslator;
import fi.otavanopisto.kuntaapi.server.persistence.model.Identifier;
import fi.otavanopisto.kuntaapi.server.settings.SystemSettingController;

@ApplicationScoped
@Singleton
@AccessTimeout (unit = TimeUnit.HOURS, value = 1l)
@SuppressWarnings ("squid:S3306")
public class PtvCodeListEntityUpdater extends EntityUpdater {

  @Inject
  private Logger logger;

  @Inject
  private SystemSettingController systemSettingController;
  
  @Inject
  private PtvCodeResourceContainer ptvCodeResourceContainer;

  @Inject
  private PtvApi ptvApi;
  
  @Inject
  private PtvTranslator ptvTranslator;
  
  @Inject
  private KuntaApiIdFactory kuntaApiIdFactory;

  @Inject
  private PtvIdFactory ptvIdFactory;

  @Inject
  private IdentifierController identifierController;

  @Inject
  private Event<IndexRequest> indexRequest;
  
  @Inject
  private PtvCodeListTaskQueue codeListTaskQueue;

  @Inject
  private ModificationHashCache modificationHashCache;

  @Override
  public String getName() {
    return "ptv-codes";
  }

  @Override
  public void timeout() {
    executeNextTask();
  }
  
  @Override
  public long getTimerInterval() {
    if (systemSettingController.inTestMode()) {
      return 5000l;
    }
    
    return super.getTimerInterval();
  }
  
  private void executeNextTask() {
    PtvCodeListTask task = codeListTaskQueue.next();
    if (task != null) {
      executeTask(task);
    } else if (codeListTaskQueue.isEmptyAndLocalNodeResponsible()) {
      fillQueue();
    }
  }
  
  private void executeTask(PtvCodeListTask task) {
    CodeListApi codeListApi = ptvApi.getCodeListApi();
    
    switch (task.getType()) {
      case BUSINESS_REGIONS:
        executeAreaCodeListTask(codeListApi, "BusinessRegions", task.getType());
      break;
      case COUNTRY:
        handleCountryCodeListResponse(codeListApi.apiV7CodeListGetCountryCodesGet(), task.getType());
      break;
      case HOSPITAL_REGIONS:
        executeAreaCodeListTask(codeListApi, "HospitalRegions", task.getType());
      break;
      case LANGUAGE:
        handleCodeListResponse(codeListApi.apiV7CodeListGetLanguageCodesGet(), task.getType());
      break;
      case MUNICIPALITY:
        handleCodeListResponse(codeListApi.apiV7CodeListGetMunicipalityCodesGet(), task.getType());
      break;
      case POSTAL:
        handleCodeListPagedResponse(codeListApi.apiV7CodeListGetPostalCodesGet(task.getPage()), task.getType());
      break;
      case PROVINCE:
        executeAreaCodeListTask(codeListApi, "Province", task.getType());
      break;
    }
    
  }

  private void executeAreaCodeListTask(CodeListApi codeListApi, String areaCodeType, CodeType codeType) {
    handleCodeListResponse(codeListApi.apiV7CodeListGetAreaCodesTypeByTypeGet(areaCodeType), codeType);
  }

  private void handleCodeListResponse(ApiResponse<List<VmOpenApiCodeListItem>> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    
    if (response.isOk()) {
      for (VmOpenApiCodeListItem item : response.getResponse()) {
        handleItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }


  private void handleCodeListPagedResponse(ApiResponse<VmOpenApiCodeListPage> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    
    if (response.isOk()) {
      for (VmOpenApiCodeListItem item : response.getResponse().getItemList()) {
        handleItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update paged code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }

  private void handleCountryCodeListResponse(ApiResponse<List<VmOpenApiDialCodeListItem>> response, CodeType codeType) {
    Long orderIndex = codeType.ordinal() * 100000l;
    if (response.isOk()) {
      for (VmOpenApiDialCodeListItem item : response.getResponse()) {
        handleDialCodeItem(orderIndex, item, codeType);
        orderIndex++;
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to update country code list [%d]: %s", response.getStatus(), response.getMessage()));
    }
  }
  
  private void handleDialCodeItem(Long orderIndex, VmOpenApiDialCodeListItem ptvItem, CodeType codeType) {
    CodeId ptvCodeId = ptvIdFactory.createCodeId(String.format("%s-%s", codeType, ptvItem.getCode()));
    Identifier identifier = identifierController.acquireIdentifier(orderIndex, ptvCodeId);
    CodeId kuntaApiCodeId = kuntaApiIdFactory.createFromIdentifier(CodeId.class, identifier);
    updateItem(orderIndex, kuntaApiCodeId, ptvTranslator.translateCode(kuntaApiCodeId, ptvItem, codeType));
  }

  private void handleItem(Long orderIndex, VmOpenApiCodeListItem ptvItem, CodeType codeType) {
    CodeId ptvCodeId = ptvIdFactory.createCodeId(String.format("%s-%s", codeType, ptvItem.getCode()));
    Identifier identifier = identifierController.acquireIdentifier(orderIndex, ptvCodeId);
    CodeId kuntaApiCodeId = kuntaApiIdFactory.createFromIdentifier(CodeId.class, identifier);
    updateItem(orderIndex, kuntaApiCodeId, ptvTranslator.translateCode(kuntaApiCodeId, ptvItem, codeType));
  }
  
  private void updateItem(Long orderIndex, CodeId kuntaApiCodeId, Code code) {
    if (code != null) {
      modificationHashCache.put(kuntaApiCodeId.getId(), createPojoHash(code));
      ptvCodeResourceContainer.put(kuntaApiCodeId, code);
      index(kuntaApiCodeId.getId(), code, orderIndex);
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to translate code %s", kuntaApiCodeId));
    }
  }
  
  private void index(String kuntaApiId, Code code, Long orderIndex) {
    String nameFi = null;
    String nameSv = null;
    String nameEn = null;
    
    for (LocalizedValue localizedValue : code.getNames()) {
      switch (localizedValue.getLanguage()) {
        case "fi":
          nameFi = localizedValue.getValue();
        break;
        case "sv":
          nameSv = localizedValue.getValue();
        break;
        case "en":
          nameEn = localizedValue.getValue();
        break;
      }
    }
    
    IndexableCode indexableCode = new IndexableCode(orderIndex, kuntaApiId, code.getType(), code.getCode(), nameFi, nameSv, nameEn);
    indexRequest.fire(new IndexRequest(indexableCode));  
  }

  private void fillQueue() {
    CodeListApi codeListApi = ptvApi.getCodeListApi();
    
    for (CodeType type : CodeType.values()) {
      if (type == CodeType.POSTAL) {
        fillQueuePostal(codeListApi);
      } else {
        codeListTaskQueue.enqueueTask(false, new PtvCodeListTask(type, 0));
      }
    }
  }

  private void fillQueuePostal(CodeListApi codeListApi) {
    ApiResponse<VmOpenApiCodeListPage> postalCodesResponse = codeListApi.apiV7CodeListGetPostalCodesGet(0);
    if (postalCodesResponse.isOk()) {
      if (postalCodesResponse.getResponse() == null) {
        logger.log(Level.SEVERE, "Failed to read PTV code list size");
      } else {
        for (int page = 0; page < postalCodesResponse.getResponse().getPageCount(); page++) {
          codeListTaskQueue.enqueueTask(false, new PtvCodeListTask(CodeType.POSTAL, page));
        }
      }
    } else {
      logger.log(Level.SEVERE, () -> String.format("Failed to read PTV code list size [%d]: %s", postalCodesResponse.getStatus(), postalCodesResponse.getMessage()));
    }
  }
  
}

{'fraction_non_alphanumeric': 0.10468485212922976, 'fraction_numerical': 0.0037172689006723884, 'mean_word_length': 4.282702858792954, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4635855', 'n_tokens_mistral': 5838, 'n_tokens_neox': 5814, 'n_words': 996}