<commit_before>package org.llc.archive.service

import org.llc.archive.domain.Mp3SermonFile
import org.llc.archive.props.CongregationPropertyLoader
import org.llc.archive.rest.domain.Sermon
import org.llc.archive.rest.domain.SyncExecution
import org.llc.archive.rest.repository.CongregationRepository
import org.llc.archive.rest.repository.MinisterRepository
import org.llc.archive.rest.repository.SermonRepository
import org.llc.archive.rest.repository.SyncExecutionRepository
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.security.core.context.SecurityContextHolder
import org.springframework.stereotype.Component

/**
 * Created by <NAME> on 4/25/15.
 */
@Component
class ArchivedSermonsServiceImpl implements ArchivedSermonsService {

    private static final Logger logger = LoggerFactory.getLogger(ArchivedSermonsServiceImpl.class);
    public static final String AMAZON_S3_URL = "https://s3.amazonaws.com"

    @Autowired
    SermonRepository sermonRepository

    @Autowired
    CongregationRepository congregationRepository

    @Autowired
    MinisterRepository ministerRepository

    @Autowired
    SyncExecutionRepository syncExecutionRepository

    @Autowired
    CongregationPropertyLoader congregationPropertyLoader

    @Autowired
    private Mp3DiscoveryService mp3DiscoveryService;

    @Override
    def updateDatastore(boolean refreshAll) {
        logger.info("> refreshing LLC sermon database");
        def start = System.currentTimeMillis()
        congregationPropertyLoader.credentials.each { name, creds ->
            logger.info("========= ${name} =========");
            updateDatastore(name, mp3DiscoveryService.processMp3Files(refreshAll, name));
        }
        def now = System.currentTimeMillis()
        def duration = now - start
        syncExecutionRepository.save(
                new SyncExecution(
                        username: SecurityContextHolder.context.authentication.principal.username,
                        date: new Date(),
                        executionTimeMs: duration
                )
        )
    }

    def updateDatastore(String name, List<Mp3SermonFile> sermons) {
        def bucket = congregationPropertyLoader.credentials[name].amazonCredentials.bucket
        sermons.each { sermon ->
            logger.debug "> sermon : ${sermon.file}"

            def minister = null
            if (sermon.minister) {
                logger.debug "> minister : ${sermon.minister}"
                def tokens = sermon.minister.split()
                if (tokens.size() == 2) {
                    def firstName = tokens[0]
                    def lastName = tokens[1]
                    minister = ministerRepository.findByFirstNameAndLastNameLike(firstName, lastName)[0]
                }
            }
            def existingSermons = sermonRepository.findByFileUrlEndingWith(sermon.file)
            if (existingSermons.size() > 0) {
                def existingSermon = existingSermons[0]
                existingSermon.bibleText = sermon.bibletext
                existingSermon.comments = sermon.comments
                existingSermon.congregation = congregationRepository.findByName(name)[0]
                existingSermon.date = sermon.date ? new Date().parse("MM/dd/yyyy", sermon.date) : null
                existingSermon.fileUrl = AMAZON_S3_URL + "/${bucket}/${sermon.file}"
                existingSermon.minister = minister ? minister.naturalName : existingSermon.minister
                sermonRepository.save(existingSermon)
            } else {

                Sermon newSermon = new Sermon(
                        minister: minister ? minister.naturalName : sermon.minister,
                        bibleText: sermon.bibletext,
                        comments: sermon.comments,
                        date: sermon.date ? new Date().parse("MM/dd/yyyy", sermon.date) : null,
                        congregation: congregationRepository.findByName(name)[0],
                        fileUrl: AMAZON_S3_URL + "/${bucket}/${sermon.file}"
                )
                sermonRepository.save(newSermon)
            }


        }
    }
}
<commit_msg>closes #51 : Delete old sync execution records<commit_after>package org.llc.archive.service

import org.llc.archive.domain.Mp3SermonFile
import org.llc.archive.props.CongregationPropertyLoader
import org.llc.archive.rest.domain.Sermon
import org.llc.archive.rest.domain.SyncExecution
import org.llc.archive.rest.repository.CongregationRepository
import org.llc.archive.rest.repository.MinisterRepository
import org.llc.archive.rest.repository.SermonRepository
import org.llc.archive.rest.repository.SyncExecutionRepository
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.security.core.context.SecurityContextHolder
import org.springframework.stereotype.Component

/**
 * Created by <NAME> on 4/25/15.
 */
@Component
class ArchivedSermonsServiceImpl implements ArchivedSermonsService {

    private static final Logger logger = LoggerFactory.getLogger(ArchivedSermonsServiceImpl.class);
    public static final String AMAZON_S3_URL = "https://s3.amazonaws.com"

    @Autowired
    SermonRepository sermonRepository

    @Autowired
    CongregationRepository congregationRepository

    @Autowired
    MinisterRepository ministerRepository

    @Autowired
    SyncExecutionRepository syncExecutionRepository

    @Autowired
    CongregationPropertyLoader congregationPropertyLoader

    @Autowired
    private Mp3DiscoveryService mp3DiscoveryService;

    @Override
    def updateDatastore(boolean refreshAll) {
        logger.info("> refreshing LLC sermon database");
        def start = System.currentTimeMillis()
        congregationPropertyLoader.credentials.each { name, creds ->
            logger.info("========= ${name} =========");
            updateDatastore(name, mp3DiscoveryService.processMp3Files(refreshAll, name));
        }
        def now = System.currentTimeMillis()
        def duration = now - start
        // delete old sync execution records from the database
        syncExecutionRepository.deleteAll()
        syncExecutionRepository.save(
                new SyncExecution(
                        username: SecurityContextHolder.context.authentication.principal.username,
                        date: new Date(),
                        executionTimeMs: duration
                )
        )
    }

    def updateDatastore(String name, List<Mp3SermonFile> sermons) {
        def bucket = congregationPropertyLoader.credentials[name].amazonCredentials.bucket
        sermons.each { sermon ->
            logger.debug "> sermon : ${sermon.file}"

            def minister = null
            if (sermon.minister) {
                logger.debug "> minister : ${sermon.minister}"
                def tokens = sermon.minister.split()
                if (tokens.size() == 2) {
                    def firstName = tokens[0]
                    def lastName = tokens[1]
                    minister = ministerRepository.findByFirstNameAndLastNameLike(firstName, lastName)[0]
                }
            }
            def existingSermons = sermonRepository.findByFileUrlEndingWith(sermon.file)
            if (existingSermons.size() > 0) {
                def existingSermon = existingSermons[0]
                existingSermon.bibleText = sermon.bibletext
                existingSermon.comments = sermon.comments
                existingSermon.congregation = congregationRepository.findByName(name)[0]
                existingSermon.date = sermon.date ? new Date().parse("MM/dd/yyyy", sermon.date) : null
                existingSermon.fileUrl = AMAZON_S3_URL + "/${bucket}/${sermon.file}"
                existingSermon.minister = minister ? minister.naturalName : existingSermon.minister
                sermonRepository.save(existingSermon)
            } else {

                Sermon newSermon = new Sermon(
                        minister: minister ? minister.naturalName : sermon.minister,
                        bibleText: sermon.bibletext,
                        comments: sermon.comments,
                        date: sermon.date ? new Date().parse("MM/dd/yyyy", sermon.date) : null,
                        congregation: congregationRepository.findByName(name)[0],
                        fileUrl: AMAZON_S3_URL + "/${bucket}/${sermon.file}"
                )
                sermonRepository.save(newSermon)
            }


        }
    }
}

{'fraction_non_alphanumeric': 0.08656751103881014, 'fraction_numerical': 0.006042296072507553, 'mean_word_length': 2.4114149821640902, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2499202', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2231, 'n_words': 407}