<commit_before>package org.ohdsi.webapi.achilles.service;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity;
import org.ohdsi.webapi.achilles.repository.AchillesCacheRepository;
import org.ohdsi.webapi.source.Source;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

@Service
public class AchillesCacheService {
    private final AchillesCacheRepository cacheRepository;

    private final ObjectMapper objectMapper;

    @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}")
    private int batchSize;

    public AchillesCacheService(AchillesCacheRepository cacheRepository,
                                ObjectMapper objectMapper) {
        this.cacheRepository = cacheRepository;
        this.objectMapper = objectMapper;
    }

    @Transactional(readOnly = true)
    public AchillesCacheEntity getCache(Source source, String cacheName) {
        return cacheRepository.findBySourceAndCacheName(source, cacheName);
    }

    @Transactional(readOnly = true)
    public List<AchillesCacheEntity> findBySourceAndNames(Source source, List<String> names) {
        return cacheRepository.findBySourceAndNames(source, names);
    }

    @Transactional(propagation = Propagation.REQUIRES_NEW)
    public AchillesCacheEntity createCache(Source source, String cacheName, Object result) throws JsonProcessingException {
        AchillesCacheEntity cacheEntity = getCache(source, cacheName);
        String cache = objectMapper.writeValueAsString(result);
        if (Objects.nonNull(cacheEntity)) {
            cacheEntity.setCache(cache);
        } else {
            cacheEntity = new AchillesCacheEntity();
            cacheEntity.setSource(source);
            cacheEntity.setCacheName(cacheName);
            cacheEntity.setCache(cache);
        }

        return cacheRepository.save(cacheEntity);
    }

    @Transactional(propagation = Propagation.REQUIRES_NEW)
    public void saveDrilldownCacheMap(Source source, String domain, Map<Integer, ObjectNode> conceptNodes) {
        Map<String, ObjectNode> nodes = new HashMap<>(batchSize);
        for (Map.Entry<Integer, ObjectNode> entry : conceptNodes.entrySet()) {
            if (nodes.size() >= batchSize) {
                createCacheEntities(source, nodes);
                nodes.clear();
            }
            Integer key = entry.getKey();
            String cacheName = getCacheName(domain, key);
            nodes.put(cacheName, entry.getValue());
        }
        createCacheEntities(source, nodes);
        nodes.clear();
    }

    private void createCacheEntities(Source source, Map<String, ObjectNode> nodes) {
        List<AchillesCacheEntity> cacheEntities = getEntities(source, nodes);
        cacheRepository.save(cacheEntities);
    }

    private List<AchillesCacheEntity> getEntities(Source source, Map<String, ObjectNode> nodes) {
        List<String> cacheNames = new ArrayList<>(nodes.keySet());
        List<AchillesCacheEntity> cacheEntities = findBySourceAndNames(source, cacheNames);
        nodes.forEach((key, value) -> {
            // check if the entity with given cache name already exists
            Optional<AchillesCacheEntity> cacheEntity = cacheEntities.stream()
                    .filter(entity -> entity.getCacheName().equals(key))
                    .findAny();
            try {
                String newValue = objectMapper.writeValueAsString(value);
                if (cacheEntity.isPresent()) {
                    // if cache entity already exists update its value
                    cacheEntity.get().setCache(newValue);
                } else {
                    // if cache entity does not exist - create new one
                    AchillesCacheEntity newEntity = new AchillesCacheEntity();
                    newEntity.setCacheName(key);
                    newEntity.setSource(source);
                    newEntity.setCache(newValue);
                    cacheEntities.add(newEntity);
                }
            } catch (JsonProcessingException e) {
                throw new RuntimeException(e);
            }
        });
        return cacheEntities;
    }

    private String getCacheName(String domain, int conceptId) {
        return String.format("drilldown_%s_%d", domain, conceptId);
    }
}
<commit_msg> WebAPI v2.11.0 : Job [warming cache] sql syntax error #2033<commit_after>package org.ohdsi.webapi.achilles.service;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity;
import org.ohdsi.webapi.achilles.repository.AchillesCacheRepository;
import org.ohdsi.webapi.source.Source;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

@Service
public class AchillesCacheService {
    private static final Logger LOG = LoggerFactory.getLogger(AchillesCacheService.class);

    private final AchillesCacheRepository cacheRepository;

    private final ObjectMapper objectMapper;

    @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}")
    private int batchSize;

    public AchillesCacheService(AchillesCacheRepository cacheRepository,
                                ObjectMapper objectMapper) {
        this.cacheRepository = cacheRepository;
        this.objectMapper = objectMapper;
    }

    @Transactional(readOnly = true)
    public AchillesCacheEntity getCache(Source source, String cacheName) {
        return cacheRepository.findBySourceAndCacheName(source, cacheName);
    }

    @Transactional(readOnly = true)
    public List<AchillesCacheEntity> findBySourceAndNames(Source source, List<String> names) {
        return cacheRepository.findBySourceAndNames(source, names);
    }

    @Transactional(propagation = Propagation.REQUIRES_NEW)
    public AchillesCacheEntity createCache(Source source, String cacheName, Object result) throws JsonProcessingException {
        AchillesCacheEntity cacheEntity = getCache(source, cacheName);
        String cache = objectMapper.writeValueAsString(result);
        if (Objects.nonNull(cacheEntity)) {
            cacheEntity.setCache(cache);
        } else {
            cacheEntity = new AchillesCacheEntity();
            cacheEntity.setSource(source);
            cacheEntity.setCacheName(cacheName);
            cacheEntity.setCache(cache);
        }

        return cacheRepository.save(cacheEntity);
    }

    @Transactional(propagation = Propagation.REQUIRES_NEW)
    public void saveDrilldownCacheMap(Source source, String domain, Map<Integer, ObjectNode> conceptNodes) {
        if (conceptNodes.isEmpty()) { // nothing to cache
            LOG.warn("Cannot cache drilldown reports for {}, domain {}. Check if result schema contains achilles results tables.",
                    source.getSourceKey(), domain);
            return;
        }

        Map<String, ObjectNode> nodes = new HashMap<>(batchSize);
        for (Map.Entry<Integer, ObjectNode> entry : conceptNodes.entrySet()) {
            if (nodes.size() >= batchSize) {
                createCacheEntities(source, nodes);
                nodes.clear();
            }
            Integer key = entry.getKey();
            String cacheName = getCacheName(domain, key);
            nodes.put(cacheName, entry.getValue());
        }
        createCacheEntities(source, nodes);
        nodes.clear();
    }

    private void createCacheEntities(Source source, Map<String, ObjectNode> nodes) {
        List<AchillesCacheEntity> cacheEntities = getEntities(source, nodes);
        cacheRepository.save(cacheEntities);
    }

    private List<AchillesCacheEntity> getEntities(Source source, Map<String, ObjectNode> nodes) {
        List<String> cacheNames = new ArrayList<>(nodes.keySet());
        List<AchillesCacheEntity> cacheEntities = findBySourceAndNames(source, cacheNames);
        nodes.forEach((key, value) -> {
            // check if the entity with given cache name already exists
            Optional<AchillesCacheEntity> cacheEntity = cacheEntities.stream()
                    .filter(entity -> entity.getCacheName().equals(key))
                    .findAny();
            try {
                String newValue = objectMapper.writeValueAsString(value);
                if (cacheEntity.isPresent()) {
                    // if cache entity already exists update its value
                    cacheEntity.get().setCache(newValue);
                } else {
                    // if cache entity does not exist - create new one
                    AchillesCacheEntity newEntity = new AchillesCacheEntity();
                    newEntity.setCacheName(key);
                    newEntity.setSource(source);
                    newEntity.setCache(newValue);
                    cacheEntities.add(newEntity);
                }
            } catch (JsonProcessingException e) {
                throw new RuntimeException(e);
            }
        });
        return cacheEntities;
    }

    private String getCacheName(String domain, int conceptId) {
        return String.format("drilldown_%s_%d", domain, conceptId);
    }
}

{'fraction_non_alphanumeric': 0.08968968968968968, 'fraction_numerical': 0.001001001001001001, 'mean_word_length': 2.8338449731389104, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2406670', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2533, 'n_words': 535}