<commit_before>#!/usr/bin/env python

import sys
import os
import errno
import time
import datetime
import hashlib
import logging
import ConfigParser
from optparse import OptionParser

import MySQLdb
import xlrd
import xlwt

NOW = time.strftime("%s")          # Epoch-time
NOWHUMAN = time.strftime("%F %T")  # Human-readable time
# Trick to establish the directory of the actual script, even when called
# via symlink.  The purpose of this is to write output-files relative to
# the script-directory, not the current directory.
SCRIPTDIR = os.path.dirname(os.path.realpath(__file__))

# Give all important columns a name
LIDNUMMER = 0
LIDSINDS = 2
LIDTOT = 3
NAAM  = 4
GEBDATUM = 6
POSTCODE = 8
EMAIL = 10
REGIO = 12
STEMRECHT = 15
# Aid to detect input-format changes
EXPECTED_INPUT_COLUMNS = 16  # Columns 0 to 15 (incl.)
# Extra columns not present in input-format
VOORNAAM = 16
ACHTERNAAM = 17
# If amount or ordering of columns in input-format changes, several
# code changes are needed:
# - change numbers of extra columns
# - change HEADER, CELL_STYLE and COLUMN_WIDTH

# Excel-output formatting
STYLE_DEFAULT = xlwt.Style.default_style
STYLE_HEADER = xlwt.easyxf("font: bold on")
STYLE_DATE = xlwt.easyxf(num_format_str="YYYY-MM-DD")
HEADER = [
    "Lidnummer",        "Lidsoort",         "Lid sinds",        "Lid beeindigd",
    "Volledige naam",   "Geslacht",         "Geboortedatum",    "Straat",
    "Postcode",         "Plaats",           "Emailadres",       "Afdeling",
    "Regio",            "Telefoonnummer",   "Mobiel",           "Stemrecht",
    "Voorna(a)m(en)",   "Achternaam",
]
CELL_STYLE = [
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DATE,         STYLE_DATE,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DATE,         STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,
]
# Every 1000 is 0.3 inch is 7.62 mm (full metal jacket)
COLUMN_WIDTH = [
    2000,               3000,               3000,               3000,
    6000,               3000,               3000,               6000,
    3000,               5000,               8000,               5000,
    5000,               4000,               4000,               2000,
    4000,               4000,
]

# All departments with their postcal code ranges
AFDELINGEN = {
    "Amsterdam":[
        (1000,2159),
        (8200,8249)],
    "Leiden-Haaglanden":[
        (2160,2799)],
    "Rotterdam":[
        (2800,3399),
        (4200,4299)],
    "Utrecht":[
        (3400,4199),
        (6700,6799),
        (7300,7399),
        (8160,8199),
        (8250,8299)],
    "Brabant":[
        (4300,5339),
        (5400,5799)],
    "Arnhem-Nijmegen":[
        (5340,5399),
        (5800,5899),
        (6500,6699),
        (6800,7299)],
    "Maastricht":[
        (5900,6499)],
    "Twente":[
        (7400,7799),
        (8100,8159)],<commit_msg>Stedendriehoek to Twente<commit_after>#!/usr/bin/env python

import sys
import os
import errno
import time
import datetime
import hashlib
import logging
import ConfigParser
from optparse import OptionParser

import MySQLdb
import xlrd
import xlwt

NOW = time.strftime("%s")          # Epoch-time
NOWHUMAN = time.strftime("%F %T")  # Human-readable time
# Trick to establish the directory of the actual script, even when called
# via symlink.  The purpose of this is to write output-files relative to
# the script-directory, not the current directory.
SCRIPTDIR = os.path.dirname(os.path.realpath(__file__))

# Give all important columns a name
LIDNUMMER = 0
LIDSINDS = 2
LIDTOT = 3
NAAM  = 4
GEBDATUM = 6
POSTCODE = 8
EMAIL = 10
REGIO = 12
STEMRECHT = 15
# Aid to detect input-format changes
EXPECTED_INPUT_COLUMNS = 16  # Columns 0 to 15 (incl.)
# Extra columns not present in input-format
VOORNAAM = 16
ACHTERNAAM = 17
# If amount or ordering of columns in input-format changes, several
# code changes are needed:
# - change numbers of extra columns
# - change HEADER, CELL_STYLE and COLUMN_WIDTH

# Excel-output formatting
STYLE_DEFAULT = xlwt.Style.default_style
STYLE_HEADER = xlwt.easyxf("font: bold on")
STYLE_DATE = xlwt.easyxf(num_format_str="YYYY-MM-DD")
HEADER = [
    "Lidnummer",        "Lidsoort",         "Lid sinds",        "Lid beeindigd",
    "Volledige naam",   "Geslacht",         "Geboortedatum",    "Straat",
    "Postcode",         "Plaats",           "Emailadres",       "Afdeling",
    "Regio",            "Telefoonnummer",   "Mobiel",           "Stemrecht",
    "Voorna(a)m(en)",   "Achternaam",
]
CELL_STYLE = [
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DATE,         STYLE_DATE,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DATE,         STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,      STYLE_DEFAULT,
    STYLE_DEFAULT,      STYLE_DEFAULT,
]
# Every 1000 is 0.3 inch is 7.62 mm (full metal jacket)
COLUMN_WIDTH = [
    2000,               3000,               3000,               3000,
    6000,               3000,               3000,               6000,
    3000,               5000,               8000,               5000,
    5000,               4000,               4000,               2000,
    4000,               4000,
]

# All departments with their postcal code ranges
AFDELINGEN = {
    "Amsterdam":[
        (1000,2159),
        (8200,8249)],
    "Leiden-Haaglanden":[
        (2160,2799)],
    "Rotterdam":[
        (2800,3399),
        (4200,4299)],
    "Utrecht":[
        (3400,4199),
        (6700,6799),
        (8250,8299)],
    "Brabant":[
        (4300,5339),
        (5400,5799)],
    "Arnhem-Nijmegen":[
        (5340,5399),
        (5800,5899),
        (6500,6699),
        (6800,6959),
        (6976,7199)],
    "Maastricht":[
        (5900,6499)],
    "Twente":[
        (6960,6975),
        (7200,7799),
        (8100,8199)],
{'fraction_non_alphanumeric': 0.10882991153396762, 'fraction_numerical': 0.08479385745284594, 'mean_word_length': 2.209426888055704, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '451236', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2018, 'n_words': 509}