<commit_before>package mingzuozhibi.action;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import mingzuozhibi.persist.disc.Disc;
import mingzuozhibi.persist.disc.Disc.DiscType;
import mingzuozhibi.service.DiscInfo;
import mingzuozhibi.service.DiscInfosSpider;
import mingzuozhibi.utils.JmsHelper;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;

import java.lang.reflect.Type;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

import static mingzuozhibi.utils.DiscUtils.needUpdateAsins;

@RestController
public class DiscSpiderController extends BaseController {

    @Autowired
    private JmsHelper jmsHelper;

    @Autowired
    private JmsTemplate jmsTemplate;

    @Autowired
    private DiscInfosSpider discInfosSpider;

    private Gson gson = new Gson();

    @Scheduled(cron = "0 59 * * * ?")
    @GetMapping("/admin/sendNeedUpdateAsins")
    public void sendNeedUpdateAsins() {
        dao.execute(session -> {
            JSONArray array = new JSONArray();
            needUpdateAsins(session).forEach(array::put);
            jmsTemplate.convertAndSend("need.update.asins", array.toString());
        });
    }

    @JmsListener(destination = "prev.update.discs")
    public void discSpiderUpdate(String json) {
        Type type = new TypeToken<ArrayList<DiscInfo>>() {}.getType();
        List<DiscInfo> discInfos = gson.fromJson(json, type);
        LOGGER.info("JMS <- prev.update.discs size=" + discInfos.size());
        discInfosSpider.updateDiscInfos(discInfos);
    }

    @Transactional
    @PreAuthorize("hasRole('BASIC')")
    @GetMapping(value = "/api/admin/fetchCount")
    public String fetchCount() {
        int fetchCount = needUpdateAsins(dao.session()).size();
        debugRequest("[fetchCount:{}]", fetchCount);
        return objectResult(fetchCount);
    }

    @Transactional
    @PreAuthorize("hasRole('BASIC')")
    @GetMapping(value = "/api/admin/searchDisc/{asin}", produces = MEDIA_TYPE)
    public String searchDisc(@PathVariable String asin) {
        Disc disc = dao.lookup(Disc.class, "asin", asin);
        if (disc != null) {
            if (LOGGER.isDebugEnabled()) {
                debugRequest("[申请查询碟片][碟片已存在于本地][ASIN={}]", asin);
            }
            return objectResult(disc.toJSON());
        }
        return searchDiscFromAmazon(asin);
    }

    private String searchDiscFromAmazon(@PathVariable String asin) {<commit_msg>feat(updater): Add update done result<commit_after>package mingzuozhibi.action;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import mingzuozhibi.persist.disc.Disc;
import mingzuozhibi.persist.disc.Disc.DiscType;
import mingzuozhibi.service.DiscInfo;
import mingzuozhibi.service.DiscInfosSpider;
import mingzuozhibi.utils.JmsHelper;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;

import java.lang.reflect.Type;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

import static mingzuozhibi.utils.DiscUtils.needUpdateAsins;

@RestController
public class DiscSpiderController extends BaseController {

    @Autowired
    private JmsHelper jmsHelper;

    @Autowired
    private JmsTemplate jmsTemplate;

    @Autowired
    private DiscInfosSpider discInfosSpider;

    private Gson gson = new Gson();

    @Scheduled(cron = "0 59 * * * ?")
    @GetMapping("/admin/sendNeedUpdateAsins")
    public void sendNeedUpdateAsins() {
        dao.execute(session -> {
            JSONArray array = new JSONArray();
            needUpdateAsins(session).forEach(array::put);
            jmsTemplate.convertAndSend("need.update.asins", array.toString());
        });
    }

    @JmsListener(destination = "prev.update.discs")
    public void discSpiderUpdate(String json) {
        Type type = new TypeToken<ArrayList<DiscInfo>>() {}.getType();
        List<DiscInfo> discInfos = gson.fromJson(json, type);
        LOGGER.info("JMS <- prev.update.discs size=" + discInfos.size());
        discInfosSpider.updateDiscInfos(discInfos);
    }

    @JmsListener(destination = "done.update.discs")
    public void discSpiderUpdate2(String json) {
        Type type = new TypeToken<ArrayList<DiscInfo>>() {}.getType();
        List<DiscInfo> discInfos = gson.fromJson(json, type);
        LOGGER.info("JMS <- done.update.discs size=" + discInfos.size());
        discInfosSpider.updateDiscInfos(discInfos);
    }

    @Transactional
    @PreAuthorize("hasRole('BASIC')")
    @GetMapping(value = "/api/admin/fetchCount")
    public String fetchCount() {
        int fetchCount = needUpdateAsins(dao.session()).size();
        debugRequest("[fetchCount:{}]", fetchCount);
        return objectResult(fetchCount);
    }

    @Transactional
    @PreAuthorize("hasRole('BASIC')")
    @GetMapping(value = "/api/admin/searchDisc/{asin}", produces = MEDIA_TYPE)
    public String searchDisc(@PathVariable String asin) {
        Disc disc = dao.lookup(Disc.class, "asin", asin);
        if (disc != null) {
            if (LOGGER.isDebugEnabled()) {
                debugRequest("[申请查询碟片][碟片已存在于本地][ASIN={}]", asin);
            }
            return objectResult(disc.toJSON());
        }
        return searchDiscFromAmazon(asin);
    }

    private String searchDiscFromAmazon(@PathVariable String asin) {
{'fraction_non_alphanumeric': 0.11789310611928737, 'fraction_numerical': 0.0010844306738962044, 'mean_word_length': 3.962336664104535, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4041983', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1953, 'n_words': 334}