<commit_before>#!/usr/bin/env python
import os
import sqlite3
import sys
from xml.etree import ElementTree


def generate_columns(table, columns):
	ret = []
	types = {
		"Bool": "bool",
		"Int": "int",
		"Long": "long",
		"ULong": "unsigned long",
		"String": "text",
		"LocString": "text",
		"AssetPath": "text",
	}
	for name, type in columns:
		cn = name.lower()
		if type == "LocString":
			cn += "_enus"
		t = "%s %s" % (cn, types[type])
		if cn == "id":
			t += " primary key"
		ret.append(t)

	return ",\n".join(ret)


def get_field(table, record, column, type):
	xpath = './Field[@column="%s"]' % (column)
	if type == "LocString":
		xpath += "/enUS"

	data = record.find(xpath)
	if data is None:
		return None
	data = data.text

	if type == "Bool":
		return True if data == "True" else False
	elif type in ("Int", "Long", "ULong"):
		if data is None:
			return None
		return int(data)
	return data


def get_tag(e, tag):
	elem = e.find('Tag[@enumID="%i"]' % (tag))
	if elem is not None:
		return int(elem.attrib["value"])
	return 0
<commit_msg>Fix DBF import for BANNER.xml post 10357<commit_after>#!/usr/bin/env python
import os
import sqlite3
import sys
from xml.etree import ElementTree


def generate_columns(table, columns):
	ret = []
	types = {
		"Bool": "bool",
		"Int": "int",
		"Long": "long",
		"ULong": "unsigned long",
		"String": "text",
		"LocString": "text",
		"AssetPath": "text",
	}
	primary_key = None
	for name, type in columns:
		cn = name.lower()
		if type == "LocString":
			cn += "_enus"
		t = "%s %s" % (cn, types[type])
		if cn in ("id", "unique_id") and not primary_key:
			t += " primary key"
			primary_key = cn
		ret.append(t)

	return ",\n".join(ret)


def get_field(table, record, column, type):
	xpath = './Field[@column="%s"]' % (column)
	if type == "LocString":
		xpath += "/enUS"

	data = record.find(xpath)
	if data is None:
		return None
	data = data.text

	if type == "Bool":
		return True if data == "True" else False
	elif type in ("Int", "Long", "ULong"):
		if data is None:
			return None
		return int(data)
	return data


def get_tag(e, tag):
	elem = e.find('Tag[@enumID="%i"]' % (tag))
	if elem is not None:
		return int(elem.attrib["value"])
	return 0

{'fraction_non_alphanumeric': 0.17967332123411978, 'fraction_numerical': 0.004083484573502722, 'mean_word_length': 3.574688796680498, 'pattern_counts': {'":': 21, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6685141', 'n_tokens_mistral': 911, 'n_tokens_neox': 840, 'n_words': 259}