<commit_before>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
dbfile

From a file given as argument executes SQL file contents to database
which is pointed out via environment variables.

This script depends on script dboperator.
"""
import sys,getopt,os
import pymssql
from time import localtime, strftime
import glob # tiedostojen listaamiseen, kelpaa myös ns wildcardsit
import re

import dboperator

def show(message):
  print strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message

def loadsql(sqlfile,verbose=False):
  fd = open(sqlfile, 'r')
  allsql = fd.read()
  fd.close()
  # split MS SQL batches
  for sql in re.split('\ngo.*\n',allsql):
    if verbose: show(sql)
    dboperator.execute(sql)

def load(sqlfile,migrate,verbose=False):
  show("begin "+sqlfile)

  number_togo = None
  if "__" in sqlfile:
    number_togo = int(sqlfile[sqlfile.rfind("/")+1:sqlfile.index("__",sqlfile.rfind("/")+1)])

  if migrate and number_togo is not None:
    # migration install step 0 is special, just add the table for others
    if number_togo==0:
      loadsql(sqlfile,verbose)
    else:
      number_last = None<commit_msg>Ignore case for "GO".<commit_after>#!/usr/bin/python
# vim: set fileencoding=UTF-8 :
"""
dbfile

From a file given as argument executes SQL file contents to database
which is pointed out via environment variables.

This script depends on script dboperator.
"""
import sys,getopt,os
import pymssql
from time import localtime, strftime
import glob # tiedostojen listaamiseen, kelpaa myös ns wildcardsit
import re

import dboperator

def show(message):
  print strftime("%Y-%m-%d %H:%M:%S", localtime())+" "+message

def loadsql(sqlfile,verbose=False):
  fd = open(sqlfile, 'r')
  allsql = fd.read()
  fd.close()
  # split MS SQL batches
  for sql in re.split('\ngo.*\n', allsql, flags=re.IGNORECASE):
    if verbose: show(sql)
    dboperator.execute(sql)

def load(sqlfile,migrate,verbose=False):
  show("begin "+sqlfile)

  number_togo = None
  if "__" in sqlfile:
    number_togo = int(sqlfile[sqlfile.rfind("/")+1:sqlfile.index("__",sqlfile.rfind("/")+1)])

  if migrate and number_togo is not None:
    # migration install step 0 is special, just add the table for others
    if number_togo==0:
      loadsql(sqlfile,verbose)
    else:
      number_last = None
{'fraction_non_alphanumeric': 0.12292938099389712, 'fraction_numerical': 0.004359197907585004, 'mean_word_length': 4.36214953271028, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905268', 'n_tokens_mistral': 789, 'n_tokens_neox': 774, 'n_words': 232}