<commit_before>#!/usr/bin/php
<?php

	/***************************************************************************/
	$typeAliasMap = array();
	$typeMap = array(
		'bi' => 'bigint',
		'ti' => 'tinyint',
		'i'  => 'int',
		'd'  => 'decimal',
		'c'  => 'char',
		'v'  => 'varchar',
		't'  => 'text',
		'dt' => 'datetime'
	);

	function d($s) { printf("%s\n", $s); }
	function closeTable($tableColumns, $primaryKeyName) {
		if (!empty($primaryKeyName)) {
			$tableColumns[] = sprintf('PRIMARY KEY (`%s`)', $primaryKeyName);
		}
	
		printf("%s\n) ENGINE=MyISAM DEFAULT CHARSET=utf8;\n\n", implode(",\n\t", $tableColumns));
	}


	/***************************************************************************/
	class ColumnInfo {
		public $type = '';
		public $length = '';
		public $default = '';
		public $isNumeric = false;
		public $isSigned = false;
		public $isNullable = false;
		public $isAutoIncrement = false;
		public $isPrimaryKey = false;
		
		public static function fromString($line) {
			global $typeMap, $typeAliasMap;
			$column = new ColumnInfo();
			
			$colonPos = strpos($line, ':');
			if ($colonPos === false) {
				return false;
			}
			
			$fixDecimal = false;
			$defaultValueOpen = false;
			$options = substr($line, 0, $colonPos);
			$column->name = substr($line, $colonPos + 1);
			$column->name = str_replace(' ', '_', $column->name);
			
			for ($i = 0; $i < strlen($options); $i++) {
				$char = $options[$i];
				if ($char == '*') {
					$column->isPrimaryKey = true;
					continue;
				}
				elseif ($char == '+') {
					$column->isAutoIncrement = true;
					continue;
				}
				elseif ($char == '-') {
					$column->isSigned = true;
					continue;
				}
				elseif ($char == '^') {
					$column->isNullable = true;
					continue;
				}
				elseif ($char == '(') {
					$defaultValueOpen = true;
					continue;
				}
				elseif ($char == ')') {
					$defaultValueOpen = false;
					continue;
				}
				elseif ((is_numeric($char) || $char == '.') && !$defaultValueOpen) {
					$column->length .= $char;
					if ($char == '.') {
						$fixDecimal = true;
					}
					continue;
				}
				
				if ($defaultValueOpen) {
					$column->default .= $char;
				}
				else {
					$column->type .= $char;
				}
			}
			
			if ($fixDecimal) {
				$bits = explode('.', $column->length);
				$column->length = ($bits[0] + $bits[1]) .','. $bits[1];
			}
			
			if (isset($typeAliasMap[$column->type])) {
				$newLine = sprintf("%s%s%s%s%s(%s):%s",
					($column->isPrimaryKey ? '*' : ''),
					($column->isAutoIncrement ? '+' : ''),
					($column->isSigned ? '-' : ''),
					($column->isNullable ? '^' : ''),
					$typeAliasMap[$column->type],
					$column->default,
					$column->name
				);
				
				return self::fromString($newLine);
			}
			
			if (!isset($typeMap[$column->type])) {
				d("ERROR: No such column type {$column->type}! ({$line})");
				break;
			}
			
			$column->type = $typeMap[$column->type];
			$column->isNumeric = preg_match('/(int|decimal|float)/', $column->type);
			
			return $column;
		}
	}
	
	/***************************************************************************/
	if ($argc != 2) {
		die("Usage: {$argv[0]} <filename>\n");
	}
	
	$inFile = $argv[1];<commit_msg>Fix bug.. vestige of the older code<commit_after>#!/usr/bin/php
<?php

	/***************************************************************************/
	$typeAliasMap = array();
	$typeMap = array(
		'bi' => 'bigint',
		'ti' => 'tinyint',
		'i'  => 'int',
		'd'  => 'decimal',
		'c'  => 'char',
		'v'  => 'varchar',
		't'  => 'text',
		'dt' => 'datetime'
	);

	function d($s) { printf("%s\n", $s); }
	function closeTable($tableColumns, $primaryKeyName) {
		if (!empty($primaryKeyName)) {
			$tableColumns[] = sprintf('PRIMARY KEY (`%s`)', $primaryKeyName);
		}
	
		printf("%s\n) ENGINE=MyISAM DEFAULT CHARSET=utf8;\n\n", implode(",\n\t", $tableColumns));
	}


	/***************************************************************************/
	class ColumnInfo {
		public $type = '';
		public $length = '';
		public $default = '';
		public $isNumeric = false;
		public $isSigned = false;
		public $isNullable = false;
		public $isAutoIncrement = false;
		public $isPrimaryKey = false;
		
		public static function fromString($line) {
			global $typeMap, $typeAliasMap;
			$column = new ColumnInfo();
			
			$colonPos = strpos($line, ':');
			if ($colonPos === false) {
				return false;
			}
			
			$fixDecimal = false;
			$defaultValueOpen = false;
			$options = substr($line, 0, $colonPos);
			$column->name = substr($line, $colonPos + 1);
			$column->name = str_replace(' ', '_', $column->name);
			
			for ($i = 0; $i < strlen($options); $i++) {
				$char = $options[$i];
				if ($char == '*') {
					$column->isPrimaryKey = true;
					continue;
				}
				elseif ($char == '+') {
					$column->isAutoIncrement = true;
					continue;
				}
				elseif ($char == '-') {
					$column->isSigned = true;
					continue;
				}
				elseif ($char == '^') {
					$column->isNullable = true;
					continue;
				}
				elseif ($char == '(') {
					$defaultValueOpen = true;
					continue;
				}
				elseif ($char == ')') {
					$defaultValueOpen = false;
					continue;
				}
				elseif ((is_numeric($char) || $char == '.') && !$defaultValueOpen) {
					$column->length .= $char;
					if ($char == '.') {
						$fixDecimal = true;
					}
					continue;
				}
				
				if ($defaultValueOpen) {
					$column->default .= $char;
				}
				else {
					$column->type .= $char;
				}
			}
			
			if ($fixDecimal) {
				$bits = explode('.', $column->length);
				$column->length = ($bits[0] + $bits[1]) .','. $bits[1];
			}
			
			if (isset($typeAliasMap[$column->type])) {
				$newLine = sprintf("%s%s%s%s%s(%s):%s",
					($column->isPrimaryKey ? '*' : ''),
					($column->isAutoIncrement ? '+' : ''),
					($column->isSigned ? '-' : ''),
					($column->isNullable ? '^' : ''),
					$typeAliasMap[$column->type],
					$column->default,
					$column->name
				);
				
				return self::fromString($newLine);
			}
			
			if (!isset($typeMap[$column->type])) {
				d("ERROR: No such column type {$column->type}! ({$line})");
				return null;
			}
			
			$column->type = $typeMap[$column->type];
			$column->isNumeric = preg_match('/(int|decimal|float)/', $column->type);
			
			return $column;
		}
	}
	
	/***************************************************************************/
	if ($argc != 2) {
		die("Usage: {$argv[0]} <filename>\n");
	}
	
	$inFile = $argv[1];
{'fraction_non_alphanumeric': 0.27636476426799006, 'fraction_numerical': 0.003101736972704715, 'mean_word_length': 2.9419315403422983, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '552140', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2187, 'n_words': 406}