<commit_before>package admin

import tli.*
import org.hibernate.ScrollMode
import java.nio.charset.Charset
import java.util.GregorianCalendar

class EnrichmentService {

  def executorService
  def ESWrapperService
  def mongoService
  def sessionFactory
  def propertyInstanceMap = org.codehaus.groovy.grails.plugins.DomainClassGrailsPlugin.PROPERTY_INSTANCE_MAP

  def runEnrhchment() {
    doLocationEnrichment()
    fillInShortCodes()
    fillInMissingUID()
  }

  def fillInShortCodes() {
    log.debug("Filling out missing short codes...");
    def entries_without_shortcodes = DirectoryEntry.executeQuery('select e.id from tli.DirectoryEntry as e where not exists ( select s from tli.DirectoryEntryShortcode as s where s.dirent = e )');
    entries_without_shortcodes.each { e ->
      def de = DirectoryEntry.get(e);
      DirectoryEntryShortcode.generateShortcode(de,de.title,true);
      log.debug("Lookup id ${e}");
    }
  }

  def fillInMissingUID() {
    log.debug("Filling out missing UIDs...");
    def entries_without_uid = DirectoryEntry.executeQuery('select e.id from tli.DirectoryEntry as e where e.uid is null or e.uid=?',['']);
    entries_without_uid.each { e ->
      def de = DirectoryEntry.get(e);
      de.uid = java.util.UUID.randomUUID().toString();
      de.save();
    }<commit_msg>Extra debug<commit_after>package admin

import tli.*
import org.hibernate.ScrollMode
import java.nio.charset.Charset
import java.util.GregorianCalendar

class EnrichmentService {

  def executorService
  def ESWrapperService
  def mongoService
  def sessionFactory
  def propertyInstanceMap = org.codehaus.groovy.grails.plugins.DomainClassGrailsPlugin.PROPERTY_INSTANCE_MAP

  def runEnrhchment() {
    doLocationEnrichment()
    fillInShortCodes()
    fillInMissingUID()
  }

  def fillInShortCodes() {
    log.debug("Filling out missing short codes...");
    def entries_without_shortcodes = DirectoryEntry.executeQuery('select e.id from tli.DirectoryEntry as e where not exists ( select s from tli.DirectoryEntryShortcode as s where s.dirent = e )');
    entries_without_shortcodes.each { e ->
      log.debug("Generate shortcode for ${e}");
      def de = DirectoryEntry.get(e);
      DirectoryEntryShortcode.generateShortcode(de,de.title,true);
      log.debug("Result: ${de.shortcodes}");
    }
  }

  def fillInMissingUID() {
    log.debug("Filling out missing UIDs...");
    def entries_without_uid = DirectoryEntry.executeQuery('select e.id from tli.DirectoryEntry as e where e.uid is null or e.uid=?',['']);
    entries_without_uid.each { e ->
      def de = DirectoryEntry.get(e);
      de.uid = java.util.UUID.randomUUID().toString();
      de.save();
    }
{'fraction_non_alphanumeric': 0.1070096942580164, 'fraction_numerical': 0.0, 'mean_word_length': 3.9410681399631677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6610791', 'n_tokens_mistral': 839, 'n_tokens_neox': 817, 'n_words': 188}