<commit_before>package com.tobykurien.sparkler.tools

import com.tobykurien.sparkler.Helper
import com.tobykurien.sparkler.db.DatabaseManager
import java.io.File
import java.io.FileReader
import org.javalite.activejdbc.Base

import static extension com.google.common.io.CharStreams.*

/**
 * Tools for working with Sparkler databases
 */
class Database {
   def init(String[] args) {
      if (args.length > 0 && args.get(0).length > 0) {
         Helper.setEnvironment(args.get(0))
      }
      
      DatabaseManager.init(Database.package.name)
      Base.open(DatabaseManager.newDataSource)
      
      var schema = new File("config/database.schema")
      var sql = new FileReader(schema).readLines.join("\r\n")
      
      Base.exec(sql)
      
      Base.close
   }
}<commit_msg>added dropping of tables in init()<commit_after>package com.tobykurien.sparkler.tools

import com.tobykurien.sparkler.Helper
import com.tobykurien.sparkler.db.DatabaseManager
import java.io.File
import java.io.FileReader
import java.sql.ResultSet
import org.javalite.activejdbc.Base

import static extension com.google.common.io.CharStreams.*

/**
 * Tools for working with Sparkler databases
 */
class Database {
   def init(String[] args) {
      if (args.length > 0 && args.get(0).length > 0) {
         Helper.setEnvironment(args.get(0))
      }
      
      DatabaseManager.init(Database.package.name)
      Base.open(DatabaseManager.newDataSource)
      Base.openTransaction

      try {
         // delete all the existing tables
         var jdbcConnection = DatabaseManager.newDataSource.connection
         var md = jdbcConnection.getMetaData();
         var rs = md.getTables(null, null, "%", #["TABLE"]);
         while (rs.next()) {
            var table = rs.getString(3)
            Base.exec("drop table " + table)
         }
         
         // load the new schema
         var schema = new File("config/database.schema")
         var sql = new FileReader(schema).readLines.join("\r\n")
         Base.exec(sql)
         Base.commitTransaction
      } catch (Exception e) {
         Base.rollbackTransaction
         throw e
      } finally {
         Base.close
      }
   }
}
{'fraction_non_alphanumeric': 0.10994044892349977, 'fraction_numerical': 0.004122766834631242, 'mean_word_length': 2.539708265802269, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6269663', 'n_tokens_mistral': 665, 'n_tokens_neox': 645, 'n_words': 135}