<commit_before>﻿Imports System.Runtime.CompilerServices
Imports Microsoft.VisualBasic.ComponentModel
Imports Microsoft.VisualBasic.Language
Imports Microsoft.VisualBasic.Linq
Imports Microsoft.VisualBasic.Text.Parser

Namespace MetaLib

    Public NotInheritable Class FormulaScanner

        Private Sub New()
        End Sub

        ' H2O
        ' (CH3)3CH
        ' (CH3)4C

        <MethodImpl(MethodImplOptions.AggressiveInlining)>
        Public Shared Function ScanFormula(formula As String) As FormulaComposition
            Return New FormulaScanner().ScanFormula(New CharPtr(formula))
        End Function

        Dim composition As New Dictionary(Of String, Counter)
        Dim buf As New List(Of Char)
        Dim digits As New List(Of Char)
        Dim formula As New List(Of Char)

        Private Sub push(c As Char)
            Dim element$ = buf.CharString

            If element.Length = 0 Then
                Return
            End If
            If Not composition.ContainsKey(element) Then
                composition(element) = New Counter
            End If

            ' 结束上一个元素
            If digits = 0 Then
                Call composition(buf.CharString).Hit()
            Else
                Call composition(buf.CharString).Add(digits.DoCall(AddressOf Val))
            End If

            formula += buf
            formula += digits
            buf *= 0
            digits *= 0

            If c <> "("c AndAlso c <> ")"c Then
                buf += c
            End If
        End Sub

        Private Function ScanFormula(scaner As CharPtr) As FormulaComposition
            Dim c As Char

            Do While Not scaner.EndRead
                c = ++scaner

                If Char.IsLetter(c) Then
                    If Char.IsUpper(c) Then
                        If buf = 0 Then
                            buf += c
                        Else
                            Call push(c)
                        End If
                    Else
                        ' 小写字母，则继续添加进入缓存
                        buf += c
                    End If
                ElseIf Char.IsDigit(c) Then
                    ' 可能是两位数或者更多
                    digits += c
                ElseIf c = "("c Then
                    ' 遇到了一个堆栈
                    ' 对这个分子基团进行函数递归
                    Dim group = scaner.DoCall(AddressOf ScanFormula)

                    Call push(c)
                    ' 后面必定会跟着数字
                    c = ++scaner

                    If Not Char.IsDigit(c) Then
                        Throw New SyntaxErrorException(scaner.RawBuffer.CharString)
                    Else
                        digits += c
                    End If<commit_msg>fix parser bugs<commit_after>﻿Imports System.Runtime.CompilerServices
Imports Microsoft.VisualBasic.ComponentModel
Imports Microsoft.VisualBasic.Language
Imports Microsoft.VisualBasic.Linq
Imports Microsoft.VisualBasic.Text.Parser

Namespace MetaLib

    Public NotInheritable Class FormulaScanner

        Private Sub New()
        End Sub

        ' H2O
        ' (CH3)3CH
        ' (CH3)4C

        <MethodImpl(MethodImplOptions.AggressiveInlining)>
        Public Shared Function ScanFormula(formula As String) As FormulaComposition
            Return New FormulaScanner().ScanFormula(New CharPtr(formula))
        End Function

        Dim composition As New Dictionary(Of String, Counter)
        Dim buf As New List(Of Char)
        Dim digits As New List(Of Char)
        Dim formula As New List(Of Char)

        Private Sub push(c As Char)
            Dim element$ = buf.CharString

            If element.Length = 0 Then
                Return
            End If
            If Not composition.ContainsKey(element) Then
                composition(element) = New Counter
            End If

            ' 结束上一个元素
            If digits = 0 Then
                Call composition(buf.CharString).Hit()
            Else
                Call composition(buf.CharString).Add(digits.DoCall(AddressOf Val))
            End If

            formula += buf
            formula += digits
            buf *= 0
            digits *= 0

            If c <> "("c AndAlso c <> ")"c Then
                buf += c
            End If
        End Sub

        Private Function ScanFormula(scaner As CharPtr) As FormulaComposition
            Dim c As Char

            Do While Not scaner.EndRead
                c = ++scaner

                If Char.IsLetter(c) Then
                    If Char.IsUpper(c) Then
                        If buf = 0 Then
                            buf += c
                        Else
                            Call push(c)
                        End If
                    Else
                        ' 小写字母，则继续添加进入缓存
                        buf += c
                    End If
                ElseIf Char.IsDigit(c) Then
                    ' 可能是两位数或者更多
                    digits += c
                ElseIf c = "("c Then
                    ' 遇到了一个堆栈
                    ' 对这个分子基团进行函数递归
                    Dim group As FormulaComposition = New FormulaScanner().ScanFormula(scaner)

                    Call push(c)
                    ' 后面必定会跟着数字
                    c = ++scaner

                    If Not Char.IsDigit(c) Then
                        Throw New SyntaxErrorException(scaner.RawBuffer.CharString)
                    Else
                        digits += c
                    End If
{'fraction_non_alphanumeric': 0.05518763796909492, 'fraction_numerical': 0.003679175864606328, 'mean_word_length': 1.1601112435439014, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6978852', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1348, 'n_words': 393}