<commit_before>{-# LANGUAGE TemplateHaskell #-}

module SchemaParser where

import qualified Text.Parsec.Token as T
import Text.ParserCombinators.Parsec
import Control.Applicative ((*>), (<*), (<$>), (<*>))
import Language.Haskell.TH
import FO.Data
import SQL.SQL
import Utils
import System.IO.Unsafe
import Data.List (partition)
import Data.Char

lexer = T.makeTokenParser T.LanguageDef {
    T.commentStart = "/*",
    T.commentEnd = "*/",
    T.commentLine = "",
    T.nestedComments = False,
    T.identStart = letter <|> char '_',
    T.identLetter = alphaNum <|> char '_',
    T.opStart = oneOf "#\'",
    T.opLetter = oneOf "",
    T.reservedNames = ["create", "table", "sequence", "not", "null", "varchar", "DEFAULT","int", "INTEGER", "INT64TYPE"],
    T.reservedOpNames = [],
    T.caseSensitive = True
}

identifier = T.identifier lexer
reserved = T.reserved lexer
reservedOp = T.reservedOp lexer
parens = T.parens lexer
integer = T.integer lexer
stringp = char '\'' *> many (noneOf "\'") <* char '\'' <* whiteSpace
comma = T.comma lexer
semi = T.semi lexer
symbol = T.symbol lexer
whiteSpace = T.whiteSpace lexer
<commit_msg>update schema parser to not depend on MissingH<commit_after>{-# LANGUAGE TemplateHaskell #-}

module SchemaParser where

import qualified Text.Parsec.Token as T
import Text.ParserCombinators.Parsec
import Control.Applicative ((*>), (<*), (<$>), (<*>))
import Language.Haskell.TH
import FO.Data
import SQL.SQL
import Utils
import System.IO.Unsafe
import Data.List (partition)
import Data.Char
import ListUtils

lexer = T.makeTokenParser T.LanguageDef {
    T.commentStart = "/*",
    T.commentEnd = "*/",
    T.commentLine = "",
    T.nestedComments = False,
    T.identStart = letter <|> char '_',
    T.identLetter = alphaNum <|> char '_',
    T.opStart = oneOf "#\'",
    T.opLetter = oneOf "",
    T.reservedNames = ["create", "table", "sequence", "not", "null", "varchar", "DEFAULT","int", "INTEGER", "INT64TYPE"],
    T.reservedOpNames = [],
    T.caseSensitive = True
}

identifier = T.identifier lexer
reserved = T.reserved lexer
reservedOp = T.reservedOp lexer
parens = T.parens lexer
integer = T.integer lexer
stringp = char '\'' *> many (noneOf "\'") <* char '\'' <* whiteSpace
comma = T.comma lexer
semi = T.semi lexer
symbol = T.symbol lexer
whiteSpace = T.whiteSpace lexer

{'fraction_non_alphanumeric': 0.15762273901808785, 'fraction_numerical': 0.0017226528854435831, 'mean_word_length': 3.942553191489362, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6676324', 'n_tokens_mistral': 828, 'n_tokens_neox': 780, 'n_words': 197}