<commit_before>import { InlineSyntaxNode } from '../../SyntaxNodes/InlineSyntaxNode'
import { PlainTextNode } from '../../SyntaxNodes/PlainTextNode'

export function getInlineNodes(text: string): InlineSyntaxNode[] {
  return [new PlainTextNode(text)]
}<commit_msg>Begin adding tokenization<commit_after>import { InlineSyntaxNode } from '../../SyntaxNodes/InlineSyntaxNode'
import { PlainTextNode } from '../../SyntaxNodes/PlainTextNode'
import { TextConsumer } from '../TextConsumer'

enum TokenMeaning {
  Text,
  EmphasisStart,
  EmphasisEnd  
}


class Token {
  constructor(public meaning: TokenMeaning, public index: number, public value: string) { }
}


export function getInlineNodes(text: string): InlineSyntaxNode[] {
  return [new PlainTextNode(text)]
}


function tokenize(text: string): Token[] {
  const consumer = new TextConsumer(text)
  const tokens: Token[] = []
  
  return tokens
}


function parse(tokens: Token[]): InlineSyntaxNode[] {
  const nodes: InlineSyntaxNode[] = []
  
  return nodes
}

{'fraction_non_alphanumeric': 0.12684365781710916, 'fraction_numerical': 0.0, 'mean_word_length': 4.784090909090909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1440417', 'n_tokens_mistral': 302, 'n_tokens_neox': 296, 'n_words': 76}