<commit_before>Attribute VB_Name = "MainMod"
Option Explicit

''
' Description here
'
'@/

'@================================================================================
' Interfaces
'@================================================================================

'@================================================================================
' Events
'@================================================================================

'@================================================================================
' Enums
'@================================================================================

Public Enum DataSources
    FromDb
    FromFile
    FromTws
End Enum

'@================================================================================
' Types
'@================================================================================

'@================================================================================
' Constants
'@================================================================================

Public Const ProjectName                            As String = "gbd"
Private Const ModuleName                            As String = "MainMod"

Private Const ContractCommand                       As String = "CONTRACT"
Private Const FromCommand                           As String = "FROM"
Private Const ToCommand                             As String = "TO"
Private Const StartCommand                          As String = "START"
Private Const StopCommand                           As String = "STOP"
Private Const NumberCommand                         As String = "NUMBER"
Private Const TimeframeCommand                      As String = "TIMEFRAME"
Private Const SessCommand                           As String = "SESS"
Private Const NonSessCommand                        As String = "NONSESS"
Private Const SessionOnlyCommand                    As String = "SESSIONONLY"
Private Const MillisecsCommand                      As String = "MILLISECS"
Private Const NoMillisecsCommand                    As String = "NOMILLISECS"
Private Const HelpCommand                           As String = "HELP"
Private Const Help1Command                          As String = "?"
Private Const SessionEndTimeCommand                 As String = "SESSIONENDTIME"
Private Const SessionStartTimeCommand               As String = "SESSIONSTARTTIME"
Private Const DateOnlyCommmand                      As String = "DATEONLY"
Private Const OutpuPathCommand                      As String = "OUTPUTPATH"
Private Const AsyncCommand                          As String = "ASYNC"
Private Const EntireSessionCommand                  As String = "ENTIRESESSION"
Private Const ExitCommand                           As String = "EXIT"
Private Const DateTimeFormatCommand                 As String = "DATETIMEFORMAT"
Private Const InFileCommand                         As String = "INFILE"
Private Const EchoCommand                           As String = "ECHO"
Private Const StartEchoCommand                      As String = "STARTECHO"
Private Const ResultFormatCommand                   As String = "RESULTFORMAT"
Private Const EchoResultFormatCommand               As String = "ECHORESULTFORMAT"

Private Const LatestParameter                       As String = "LATEST"
Private Const TodayParameter                        As String = "TODAY"
Private Const TomorrowParameter                     As String = "TOMORROW"
Private Const YesterdayParameter                    As String = "YESTERDAY"
Private Const EndOfWeekParameter                    As String = "ENDOFWEEK"
Private Const StartOfWeekParameter                  As String = "STARTOFWEEK"
Private Const StartOfPreviousWeekParameter          As String = "STARTOFPREVIOUSWEEK"
Private Const DateFormatRawParameter                As String = "RAW"
Private Const DateFormatISOParameter                As String = "ISO"
Private Const DateFormatLocalParameter              As String = "LOCAL"
Private Const OutputFormatTimestampParameter        As String = "T"
Private Const OutputFormatOpenParameter             As String = "O"
Private Const OutputFormatHighParameter             As String = "H"
Private Const OutputFormatLowParameter              As String = "L"
Private Const OutputFormatCloseParameter            As String = "C"
Private Const OutputFormatVolumeParameter           As String = "V"
Private Const OutputFormatTickVolumeParameter       As String = "TV"
Private Const OutputFormatOpenInterestParameter     As String = "OI"

Private Const AppendOperator                        As String = ">>"
Private Const OverwriteOperator                     As String = ">"

Private Const ContractVariable                      As String = "$CONTRACT"
Private Const SymbolVariable                        As String = "$SYMBOL"
Private Const LocalSymbolVariable                   As String = "$LOCALSYMBOL"
Private Const SecTypeVariable                       As String = "$SECTYPE"
Private Const SecTypeAbbrvVariable                  As String = "$SECTYPEABBRV"
Private Const ExchangeVariable                      As String = "$EXCHANGE"
Private Const ExpiryVariable                        As String = "$EXPIRY"
Private Const CurrencyVariable                      As String = "$CURRENCY"
Private Const MultiplierVariable                    As String = "$MULTIPLIER"
Private Const StrikeVariable                        As String = "$STRIKE"
Private Const RightVariable                         As String = "$RIGHT"
Private Const TodayVariable                         As String = "$TODAY"
Private Const YesterdayVariable                     As String = "$YESTERDAY"
Private Const FromDateVariable                      As String = "$FROMDATE"
Private Const FromDateTimeVariable                  As String = "$FROMDATETIME"
Private Const FromTimeVariable                      As String = "$FROMTIME"
Private Const ToDateVariable                        As String = "$TODATE"
Private Const ToDateTimeVariable                    As String = "$TODATETIME"
Private Const ToTimeVariable                        As String = "$TOTIME"
Private Const TimeframeVariable                     As String = "$TIMEFRAME"
Private Const NumberRequestedVariable               As String = "$NUMBERREQUESTED"
Private Const NumberReturnedVariable                As String = "$NUMBERRETURNED"
Private Const NewlineVariable                       As String = "$NEWLINE"

Private Const BarNumberVariable                     As String = "$BARNUMBER"
Private Const TimestampVariable                     As String = "$TIMESTAMP"
Private Const OpenVariable                          As String = "$OPEN"
Private Const HighVariable                          As String = "$HIGH"
Private Const LowVariable                           As String = "$LOW"
Private Const CloseVariable                         As String = "$CLOSE"
Private Const VolumeVariable                        As String = "$VOLUME"
Private Const TickVolumeVariable                    As String = "$TICKVOLUME"


Private Const SwitchCommandSeparator                As String = "SEP"
Private Const SwitchFromDb                          As String = "fromdb"
Private Const SwitchFromFile                        As String = "fromfile"
Private Const SwitchFromTws                         As String = "fromtws"
Private Const SwitchLogToConsole                    As String = "logtoconsole"
Private Const SwitchOutputPath                      As String = "outputpath"
Private Const SwitchApiMessageLogging               As String = "APIMESSAGELOGGING"

Private Const DefaultClientId                       As Long = 205644991

Private Const Time235900                            As Double = 0.99930556712963

Private Const FilenameCharsPattern                  As String = "^[^/\*\?""<>|]*$"
Private Const SubstitutionVariablePattern           As String = "(?:{(\$\w*)})"

Private Const DefaultResultFormat                   As String = "{" & TimestampVariable & "}," & _
                                                                "{" & OpenVariable & "}," & _
                                                                "{" & HighVariable & "}," & _
                                                                "{" & LowVariable & "}," & _
                                                                "{" & CloseVariable & "}," & _
                                                                "{" & VolumeVariable & "}," & _
                                                                "{" & TickVolumeVariable & "}"

'@================================================================================
' Member variables
'@================================================================================

Public gCon                                         As Console

Public gLogToConsole                                As Boolean

Private mClp                                        As CommandLineParser

Private mCommandSeparator                           As String

Private mAsync                                      As Boolean
Private mEntireSession                              As Boolean

Private mProcessors                                 As New EnumerableCollection
Private mCurrentProcessor                           As IProcessor

Private mDataSource                                 As DataSources

Private mTickfileName                               As String

Private mLineNumber                                 As Long
Private mContractSpec                               As IContractSpecifier
Private mFrom                                       As Date
Private mTo                                         As Date
Private mNumber                                     As Long
Private mTimePeriod                                 As TimePeriod
Private mSessionOnly                                As Boolean

Private mFatalErrorHandler                          As FatalErrorHandler

Private mIncludeMillisecs                           As Boolean

Private mHistDataStore                              As IHistoricalDataStore
Private mContractStore                              As IContractStore

Private mSessionEndTime                             As Date
Private mSessionStartTime                           As Date

Private mNormaliseDailyBarTimestamps                As Boolean

Private mTWSConnectionMonitor                       As TWSConnectionMonitor

Private mProviderReady                              As Boolean

Private mOutputPath                                 As String

Private mFilenameSubstitutionVariables()            As String
Private mMaxFilenameVariablesIndex                  As Long

Private mResultSubstitutionVariables()              As String
Private mMaxResultVariablesIndex                    As Long

Private mTimestampFormat                            As TimestampFormats
Private mTimestampDateOnlyFormat                    As TimestampFormats
Private mTimestampTimeOnlyFormat                    As TimestampFormats

Private mErorStringBuilder                          As StringBuilder

Private mStartResultFormat                           As String
Private mStartEchoResultFormat                       As String

Private mEchoToStdOut                               As Boolean

'@================================================================================
' Class Event Handlers
'@================================================================================

'@================================================================================
' XXXX Interface Members
'@================================================================================

'@================================================================================
' XXXX Event Handlers
'@================================================================================

'@================================================================================
' Properties
'@================================================================================

Public Property Get gFileSystemObject() As FileSystemObject
Static sFSO As FileSystemObject
If sFSO Is Nothing Then Set sFSO = New FileSystemObject
Set gFileSystemObject = sFSO
End Property

Public Property Get gLogger() As FormattingLogger
Static sLogger As FormattingLogger
If sLogger Is Nothing Then Set sLogger = CreateFormattingLogger("gbd", ProjectName)
Set gLogger = sLogger
End Property

Public Property Get gRegExp() As RegExp
Static sRegexp As RegExp
If sRegexp Is Nothing Then Set sRegexp = New RegExp
Set gRegExp = sRegexp
End Property

'@================================================================================
' Methods
'@================================================================================

Public Function gCreateOutputStream( _
                ByVal pOutputPath As String, _
                ByVal pOutputFilename As String, _
                ByVal pProcessor As IProcessor, _
                ByVal pAppend As Boolean, _
                ByRef pMessage As String) As TextStream
Const ProcName As String = "gCreateOutputStream"
On Error GoTo Err

pOutputPath = gPerformFilenameVariableSubstitution(pOutputPath, pProcessor)
pOutputFilename = gPerformFilenameVariableSubstitution(pOutputFilename, pProcessor)
Dim lFilename As String
lFilename = gFileSystemObject.BuildPath(pOutputPath, pOutputFilename)
Set gCreateOutputStream = CreateWriteableTextFile( _
                                    lFilename, _
                                    pOverwrite:=(Not pAppend), _
                                    pUnicode:=True)

Exit Function

Err:
If Err.Number = 52 Then
    pMessage = "Invalid filename: " & lFilename
ElseIf Err.Number = ErrorCodes.ErrSecurityException Then
    pMessage = "Couldn't create file: " & lFilename & ": " & Err.Description
Else
    gHandleUnexpectedError ProcName, ModuleName
End If
End Function

Public Function gGetContractName(ByVal pContractSpec As IContractSpecifier) As String
AssertArgument Not pContractSpec Is Nothing
gGetContractName = pContractSpec.LocalSymbol & _
                    IIf(pContractSpec.Exchange <> "", "@" & pContractSpec.Exchange, "")
End Function

Public Sub gHandleFatalError(ev As ErrorEventData)
On Error Resume Next    ' ignore any further errors that might arise

gWriteErrorString "Error "
gWriteErrorString CStr(ev.ErrorCode)
gWriteErrorString ": "
gWriteErrorLine ev.ErrorMessage
gWriteErrorLine "At:"
gWriteErrorLine ev.ErrorSource
End Sub

Public Sub gHandleUnexpectedError( _
                ByRef pProcedureName As String, _
                ByRef pModuleName As String, _
                Optional ByRef pFailpoint As String, _
                Optional ByVal pReRaise As Boolean = True, _
                Optional ByVal pLog As Boolean = False, _
                Optional ByVal pErrorNumber As Long, _
                Optional ByRef pErrorDesc As String, _
                Optional ByRef pErrorSource As String)
Dim errSource As String: errSource = IIf(pErrorSource <> "", pErrorSource, Err.Source)
Dim errDesc As String: errDesc = IIf(pErrorDesc <> "", pErrorDesc, Err.Description)
Dim errNum As Long: errNum = IIf(pErrorNumber <> 0, pErrorNumber, Err.Number)

HandleUnexpectedError pProcedureName, ProjectName, pModuleName, pFailpoint, pReRaise, pLog, errNum, errDesc, errSource
End Sub

Public Sub gNotifyUnhandledError( _
                ByRef pProcedureName As String, _
                ByRef pModuleName As String, _
                Optional ByRef pFailpoint As String, _
                Optional ByVal pErrorNumber As Long, _
                Optional ByRef pErrorDesc As String, _
                Optional ByRef pErrorSource As String)
Dim errSource As String: errSource = IIf(pErrorSource <> "", pErrorSource, Err.Source)
Dim errDesc As String: errDesc = IIf(pErrorDesc <> "", pErrorDesc, Err.Description)
Dim errNum As Long: errNum = IIf(pErrorNumber <> 0, pErrorNumber, Err.Number)

UnhandledErrorHandler.Notify pProcedureName, pModuleName, ProjectName, pFailpoint, errNum, errDesc, errSource
End Sub

Public Sub gNotifyAPIConnectionStateChange( _
                ByVal pState As ApiConnectionStates, _
                ByVal pMessage As String)
Const ProcName As String = "gNotifyAPIConnectionStateChange"
On Error GoTo Err

Select Case pState
Case ApiConnNotConnected
    gWriteLineToConsole "Not connected to TWS: " & pMessage
    mProviderReady = False
Case ApiConnConnecting
    gWriteLineToConsole "Connecting to TWS: " & pMessage
    mProviderReady = False
Case ApiConnConnected
    gWriteLineToConsole "Connected to TWS: " & pMessage
    mProviderReady = True
Case ApiConnFailed
    gWriteLineToConsole "Connection to TWS failed: " & pMessage
    mProviderReady = False
End Select

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyDataRetrieved(ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyDataRetrieved"
On Error GoTo Err

LogMessage "Data retrieved from source for contract: " & gGetContractName(pProcessor.ContractSpec)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchCancelled( _
                ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchCancelled"
On Error GoTo Err

If pProcessor.ContractSpec Is Nothing Then
    gWriteLineToConsole "Fetch cancelled for contract " & pProcessor.DataSourceName
Else
    gWriteLineToConsole "Fetch cancelled for contract " & gGetContractName(pProcessor.ContractSpec)
End If
If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchCompleted(ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchCompleted"
On Error GoTo Err

gWriteLineToConsole "Fetch completed: " & _
            pProcessor.NumberOfBarsOutput & _
            " bars for contract: " & _
            gGetContractName(pProcessor.ContractSpec)

If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchFailed( _
                ByVal pProcessor As IProcessor, _
                ByVal pErrorMessage As String)
Const ProcName As String = "gNotifyFetchFailed"
On Error GoTo Err

gWriteLineToConsole "Fetch failed for " & pProcessor.DataSourceName & ": " & pErrorMessage
If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchStarted( _
                ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchStarted"
On Error GoTo Err

LogMessage "Fetch started for contract " & gGetContractName(pProcessor.ContractSpec)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyIBServerConnectionClosed()
gWriteLineToConsole "Connection from TWS to IB servers closed"
mProviderReady = False
End Sub

Public Sub gNotifyIBServerConnectionRecovered( _
                ByVal pDataLost As Boolean)
gWriteLineToConsole "Connection from TWS to IB servers recovered"
mProviderReady = True
End Sub

Public Sub gOutputBarToStdOut( _
                ByVal pUseEchoResultFormat As Boolean, _
                ByVal pProcessor As IProcessor, _
                ByVal pBar As Bar, _
                ByVal pTimeframe As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pIncludeMillisecs As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double)
Const ProcName As String = "gOutputBarToStdOut"
On Error GoTo Err

If pBar Is Nothing Then Exit Sub

gWriteLineToStdOut formatBar(pUseEchoResultFormat, pProcessor, pBar, pBarnumber, pTimeframe, pNormaliseDailyBarTimestamps, pSecType, pTickSize, pIncludeMillisecs)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gOutputBarToTextStream( _
                ByVal pProcessor As IProcessor, _
                ByVal pBar As Bar, _
                ByVal pTimeframe As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pIncludeMillisecs As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double, _
                ByVal pStream As TextStream)
Const ProcName As String = "gOutputBarToStdOut"
On Error GoTo Err

If pBar Is Nothing Then Exit Sub

pStream.WriteLine formatBar(False, pProcessor, pBar, pBarnumber, pTimeframe, pNormaliseDailyBarTimestamps, pSecType, pTickSize, pIncludeMillisecs)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Function gPerformFilenameVariableSubstitution( _
                ByVal pString As String, _
                ByVal pProcessor As IProcessor) As String
Const ProcName As String = "gPerformFilenameVariableSubstitution"
On Error GoTo Err

Dim lContractSpec As IContractSpecifier
Set lContractSpec = pProcessor.ContractSpec

Dim lRegExp As RegExp: Set lRegExp = gRegExp
lRegExp.IgnoreCase = True

lRegExp.Pattern = SubstitutionVariablePattern
lRegExp.Global = True

Dim lMatches As MatchCollection
Set lMatches = lRegExp.Execute(pString)

Dim s As String
Dim lCurrPosn As Long: lCurrPosn = 1

Dim lMatch As Match
For Each lMatch In lMatches
    s = s & Mid$(pString, lCurrPosn, lMatch.FirstIndex - lCurrPosn + 1)
    lCurrPosn = lMatch.FirstIndex + lMatch.Length + 1
    
    Dim r As String
    
    Dim lVariable As String: lVariable = UCase$(lMatch.SubMatches(0))
    Select Case lVariable
    Case ContractVariable
        r = gGetContractName(lContractSpec)
    Case SymbolVariable
        r = lContractSpec.Symbol
    Case LocalSymbolVariable
        r = lContractSpec.LocalSymbol
    Case SecTypeVariable
        r = SecTypeToString(lContractSpec.SecType)
    Case SecTypeAbbrvVariable
        r = SecTypeToShortString(lContractSpec.SecType)
    Case ExchangeVariable
        r = lContractSpec.Exchange
    Case ExpiryVariable
        r = lContractSpec.Expiry
    Case CurrencyVariable
        r = lContractSpec.CurrencyCode
    Case MultiplierVariable
        r = lContractSpec.Multiplier
    Case StrikeVariable
        r = lContractSpec.Strike
    Case RightVariable
        r = OptionRightToString(lContractSpec.Right)
    Case TodayVariable
        r = FormatTimestamp(todayDate, mTimestampDateOnlyFormat)
    Case YesterdayVariable
        r = FormatTimestamp(yesterdayDate, mTimestampDateOnlyFormat)
    Case FromDateVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampDateOnlyFormat)
    Case FromDateTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampFormat + TimestampNoMillisecs)
    Case FromTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
    Case ToDateVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampDateOnlyFormat)
        End If
    Case ToDateTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampFormat + TimestampNoMillisecs)
        End If
    Case ToTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
        End If
    Case TimeframeVariable
        r = pProcessor.Timeframe.ToShortString
    Case NumberRequestedVariable
        r = pProcessor.NumberOfBarsRequested
    Case NumberReturnedVariable
        r = pProcessor.NumberOfBarsOutput
    Case Default
        Assert False, "Unexpected substitution variable: " & lVariable
    End Select
    s = s & escapeNonFilenameChars(r)
Next

gPerformFilenameVariableSubstitution = s & Right$(pString, Len(pString) - lCurrPosn + 1)

Exit Function

Err:
gHandleUnexpectedError ProcName, ModuleName
End Function

Public Function gPerformResultVariableSubstitution( _
                ByVal pProcessor As IProcessor, _
                ByVal pString As String, _
                ByVal pBar As Bar, _
                ByVal pTimePeriod As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double, _
                ByVal pIncludeMillisecs As Boolean) As String
Const ProcName As String = "gPerformResultVariableSubstitution"
On Error GoTo Err

Dim lContractSpec As IContractSpecifier
Set lContractSpec = pProcessor.ContractSpec

Dim lRegExp As RegExp: Set lRegExp = gRegExp
lRegExp.IgnoreCase = True

lRegExp.Pattern = SubstitutionVariablePattern
lRegExp.Global = True

Dim lMatches As MatchCollection
Set lMatches = lRegExp.Execute(pString)

Dim s As String
Dim lCurrPosn As Long: lCurrPosn = 1

Dim lMatch As Match
For Each lMatch In lMatches
    s = s & Mid$(pString, lCurrPosn, lMatch.FirstIndex - lCurrPosn + 1)
    lCurrPosn = lMatch.FirstIndex + lMatch.Length + 1
    
    Dim r As String
    
    Dim lVariable As String: lVariable = UCase$(lMatch.SubMatches(0))
    Select Case lVariable
    Case BarNumberVariable
        r = CStr(pBarnumber)
    Case TimestampVariable
        If Not pNormaliseDailyBarTimestamps Then
            r = FormatTimestamp(pBar.TimeStamp, mTimestampFormat Or (Not pIncludeMillisecs And TimestampNoMillisecs))
        ElseIf pTimePeriod.Units = TimePeriodDay Or _
                pTimePeriod.Units = TimePeriodWeek Or _
                pTimePeriod.Units = TimePeriodMonth Or _
                pTimePeriod.Units = TimePeriodYear Then
            r = FormatTimestamp(pBar.TimeStamp, mTimestampDateOnlyFormat)
        Else
            r = FormatTimestamp(pBar.TimeStamp, mTimestampFormat Or (Not pIncludeMillisecs And TimestampNoMillisecs))
        End If
    Case OpenVariable
        r = FormatPrice(pBar.OpenValue, pSecType, pTickSize)
    Case HighVariable
        r = FormatPrice(pBar.HighValue, pSecType, pTickSize)
    Case LowVariable
        r = FormatPrice(pBar.LowValue, pSecType, pTickSize)
    Case CloseVariable
        r = FormatPrice(pBar.CloseValue, pSecType, pTickSize)
    Case VolumeVariable
        r = CStr(pBar.Volume)
    Case TickVolumeVariable
        r = CStr(pBar.TickVolume)
    Case ContractVariable
        r = gGetContractName(lContractSpec)
    Case SymbolVariable
        r = lContractSpec.Symbol
    Case LocalSymbolVariable
        r = lContractSpec.LocalSymbol
    Case SecTypeVariable
        r = SecTypeToString(lContractSpec.SecType)
    Case SecTypeAbbrvVariable
        r = SecTypeToShortString(lContractSpec.SecType)
    Case ExchangeVariable
        r = lContractSpec.Exchange
    Case ExpiryVariable
        r = lContractSpec.Expiry
    Case CurrencyVariable
        r = lContractSpec.CurrencyCode
    Case MultiplierVariable
        r = lContractSpec.Multiplier
    Case StrikeVariable
        r = lContractSpec.Strike
    Case RightVariable
        r = OptionRightToString(lContractSpec.Right)
    Case FromDateVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampDateOnlyFormat)
    Case FromDateTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampFormat + TimestampNoMillisecs)
    Case FromTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
    Case ToDateVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampDateOnlyFormat)
        End If
    Case ToDateTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampFormat + TimestampNoMillisecs)
        End If
    Case ToTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
        End If
    Case TimeframeVariable
        r = pProcessor.Timeframe.ToShortString
    Case NumberRequestedVariable
        r = pProcessor.NumberOfBarsRequested
    Case NumberReturnedVariable
        r = pProcessor.NumberOfBarsOutput
    Case NewlineVariable
        r = vbCrLf
    Case Default
        Assert False, "Unexpected substitution variable: " & lVariable
    End Select
    s = s & r
Next

gPerformResultVariableSubstitution = s & Right$(pString, Len(pString) - lCurrPosn + 1)

Exit Function

Err:
gHandleUnexpectedError ProcName, ModuleName
End Function

Public Sub gWriteErrorLine( _
                ByVal pMessage As String)
Dim s As String: s = "Line " & mLineNumber & ": " & pMessage
gCon.WriteErrorLine s
mErorStringBuilder.Append s
LogMessage "StdErr: " & mErorStringBuilder.ToString
mErorStringBuilder.Clear
End Sub

Public Sub gWriteErrorString( _
                ByVal pMessage As String)
Dim s As String: s = "Line " & mLineNumber & ": " & pMessage
gCon.WriteErrorString s
mErorStringBuilder.Append s
End Sub

Public Sub gWriteLineToConsole( _
                ByVal pMessage As String)
gCon.WriteLineToConsole pMessage
LogMessage pMessage
End Sub

Public Sub gWriteLineToStdOut(ByVal pMessage As String)
Const ProcName As String = "gWriteLineToStdOut"

If pMessage <> "" Then LogMessage "StdOut: " & pMessage
gCon.WriteLine pMessage
End Sub

Public Sub Main()
On Error GoTo Err

Set mErorStringBuilder = CreateStringBuilder

Set mFatalErrorHandler = New FatalErrorHandler
ApplicationGroupName = "TradeWright"
ApplicationName = "gbd"
SetupDefaultLogging Command

Set gCon = GetConsole

logProgramId

mNumber = &H7FFFFFFF
mTo = MaxDate
mNormaliseDailyBarTimestamps = True

mTimestampFormat = TimestampDateAndTimeISO8601
mTimestampDateOnlyFormat = TimestampDateOnlyISO8601
mTimestampTimeOnlyFormat = TimestampTimeOnlyISO8601

mCommandSeparator = ";"

Set mClp = CreateCommandLineParser(Command)<commit_msg>Remove logging of retrieved bars in gbd<commit_after>Attribute VB_Name = "MainMod"
Option Explicit

''
' Description here
'
'@/

'@================================================================================
' Interfaces
'@================================================================================

'@================================================================================
' Events
'@================================================================================

'@================================================================================
' Enums
'@================================================================================

Public Enum DataSources
    FromDb
    FromFile
    FromTws
End Enum

'@================================================================================
' Types
'@================================================================================

'@================================================================================
' Constants
'@================================================================================

Public Const ProjectName                            As String = "gbd"
Private Const ModuleName                            As String = "MainMod"

Private Const ContractCommand                       As String = "CONTRACT"
Private Const FromCommand                           As String = "FROM"
Private Const ToCommand                             As String = "TO"
Private Const StartCommand                          As String = "START"
Private Const StopCommand                           As String = "STOP"
Private Const NumberCommand                         As String = "NUMBER"
Private Const TimeframeCommand                      As String = "TIMEFRAME"
Private Const SessCommand                           As String = "SESS"
Private Const NonSessCommand                        As String = "NONSESS"
Private Const SessionOnlyCommand                    As String = "SESSIONONLY"
Private Const MillisecsCommand                      As String = "MILLISECS"
Private Const NoMillisecsCommand                    As String = "NOMILLISECS"
Private Const HelpCommand                           As String = "HELP"
Private Const Help1Command                          As String = "?"
Private Const SessionEndTimeCommand                 As String = "SESSIONENDTIME"
Private Const SessionStartTimeCommand               As String = "SESSIONSTARTTIME"
Private Const DateOnlyCommmand                      As String = "DATEONLY"
Private Const OutpuPathCommand                      As String = "OUTPUTPATH"
Private Const AsyncCommand                          As String = "ASYNC"
Private Const EntireSessionCommand                  As String = "ENTIRESESSION"
Private Const ExitCommand                           As String = "EXIT"
Private Const DateTimeFormatCommand                 As String = "DATETIMEFORMAT"
Private Const InFileCommand                         As String = "INFILE"
Private Const EchoCommand                           As String = "ECHO"
Private Const StartEchoCommand                      As String = "STARTECHO"
Private Const ResultFormatCommand                   As String = "RESULTFORMAT"
Private Const EchoResultFormatCommand               As String = "ECHORESULTFORMAT"

Private Const LatestParameter                       As String = "LATEST"
Private Const TodayParameter                        As String = "TODAY"
Private Const TomorrowParameter                     As String = "TOMORROW"
Private Const YesterdayParameter                    As String = "YESTERDAY"
Private Const EndOfWeekParameter                    As String = "ENDOFWEEK"
Private Const StartOfWeekParameter                  As String = "STARTOFWEEK"
Private Const StartOfPreviousWeekParameter          As String = "STARTOFPREVIOUSWEEK"
Private Const DateFormatRawParameter                As String = "RAW"
Private Const DateFormatISOParameter                As String = "ISO"
Private Const DateFormatLocalParameter              As String = "LOCAL"
Private Const OutputFormatTimestampParameter        As String = "T"
Private Const OutputFormatOpenParameter             As String = "O"
Private Const OutputFormatHighParameter             As String = "H"
Private Const OutputFormatLowParameter              As String = "L"
Private Const OutputFormatCloseParameter            As String = "C"
Private Const OutputFormatVolumeParameter           As String = "V"
Private Const OutputFormatTickVolumeParameter       As String = "TV"
Private Const OutputFormatOpenInterestParameter     As String = "OI"

Private Const AppendOperator                        As String = ">>"
Private Const OverwriteOperator                     As String = ">"

Private Const ContractVariable                      As String = "$CONTRACT"
Private Const SymbolVariable                        As String = "$SYMBOL"
Private Const LocalSymbolVariable                   As String = "$LOCALSYMBOL"
Private Const SecTypeVariable                       As String = "$SECTYPE"
Private Const SecTypeAbbrvVariable                  As String = "$SECTYPEABBRV"
Private Const ExchangeVariable                      As String = "$EXCHANGE"
Private Const ExpiryVariable                        As String = "$EXPIRY"
Private Const CurrencyVariable                      As String = "$CURRENCY"
Private Const MultiplierVariable                    As String = "$MULTIPLIER"
Private Const StrikeVariable                        As String = "$STRIKE"
Private Const RightVariable                         As String = "$RIGHT"
Private Const TodayVariable                         As String = "$TODAY"
Private Const YesterdayVariable                     As String = "$YESTERDAY"
Private Const FromDateVariable                      As String = "$FROMDATE"
Private Const FromDateTimeVariable                  As String = "$FROMDATETIME"
Private Const FromTimeVariable                      As String = "$FROMTIME"
Private Const ToDateVariable                        As String = "$TODATE"
Private Const ToDateTimeVariable                    As String = "$TODATETIME"
Private Const ToTimeVariable                        As String = "$TOTIME"
Private Const TimeframeVariable                     As String = "$TIMEFRAME"
Private Const NumberRequestedVariable               As String = "$NUMBERREQUESTED"
Private Const NumberReturnedVariable                As String = "$NUMBERRETURNED"
Private Const NewlineVariable                       As String = "$NEWLINE"

Private Const BarNumberVariable                     As String = "$BARNUMBER"
Private Const TimestampVariable                     As String = "$TIMESTAMP"
Private Const OpenVariable                          As String = "$OPEN"
Private Const HighVariable                          As String = "$HIGH"
Private Const LowVariable                           As String = "$LOW"
Private Const CloseVariable                         As String = "$CLOSE"
Private Const VolumeVariable                        As String = "$VOLUME"
Private Const TickVolumeVariable                    As String = "$TICKVOLUME"


Private Const SwitchCommandSeparator                As String = "SEP"
Private Const SwitchFromDb                          As String = "fromdb"
Private Const SwitchFromFile                        As String = "fromfile"
Private Const SwitchFromTws                         As String = "fromtws"
Private Const SwitchLogToConsole                    As String = "logtoconsole"
Private Const SwitchOutputPath                      As String = "outputpath"
Private Const SwitchApiMessageLogging               As String = "APIMESSAGELOGGING"

Private Const DefaultClientId                       As Long = 205644991

Private Const Time235900                            As Double = 0.99930556712963

Private Const FilenameCharsPattern                  As String = "^[^/\*\?""<>|]*$"
Private Const SubstitutionVariablePattern           As String = "(?:{(\$\w*)})"

Private Const DefaultResultFormat                   As String = "{" & TimestampVariable & "}," & _
                                                                "{" & OpenVariable & "}," & _
                                                                "{" & HighVariable & "}," & _
                                                                "{" & LowVariable & "}," & _
                                                                "{" & CloseVariable & "}," & _
                                                                "{" & VolumeVariable & "}," & _
                                                                "{" & TickVolumeVariable & "}"

'@================================================================================
' Member variables
'@================================================================================

Public gCon                                         As Console

Public gLogToConsole                                As Boolean

Private mClp                                        As CommandLineParser

Private mCommandSeparator                           As String

Private mAsync                                      As Boolean
Private mEntireSession                              As Boolean

Private mProcessors                                 As New EnumerableCollection
Private mCurrentProcessor                           As IProcessor

Private mDataSource                                 As DataSources

Private mTickfileName                               As String

Private mLineNumber                                 As Long
Private mContractSpec                               As IContractSpecifier
Private mFrom                                       As Date
Private mTo                                         As Date
Private mNumber                                     As Long
Private mTimePeriod                                 As TimePeriod
Private mSessionOnly                                As Boolean

Private mFatalErrorHandler                          As FatalErrorHandler

Private mIncludeMillisecs                           As Boolean

Private mHistDataStore                              As IHistoricalDataStore
Private mContractStore                              As IContractStore

Private mSessionEndTime                             As Date
Private mSessionStartTime                           As Date

Private mNormaliseDailyBarTimestamps                As Boolean

Private mTWSConnectionMonitor                       As TWSConnectionMonitor

Private mProviderReady                              As Boolean

Private mOutputPath                                 As String

Private mFilenameSubstitutionVariables()            As String
Private mMaxFilenameVariablesIndex                  As Long

Private mResultSubstitutionVariables()              As String
Private mMaxResultVariablesIndex                    As Long

Private mTimestampFormat                            As TimestampFormats
Private mTimestampDateOnlyFormat                    As TimestampFormats
Private mTimestampTimeOnlyFormat                    As TimestampFormats

Private mErorStringBuilder                          As StringBuilder

Private mStartResultFormat                           As String
Private mStartEchoResultFormat                       As String

Private mEchoToStdOut                               As Boolean

'@================================================================================
' Class Event Handlers
'@================================================================================

'@================================================================================
' XXXX Interface Members
'@================================================================================

'@================================================================================
' XXXX Event Handlers
'@================================================================================

'@================================================================================
' Properties
'@================================================================================

Public Property Get gFileSystemObject() As FileSystemObject
Static sFSO As FileSystemObject
If sFSO Is Nothing Then Set sFSO = New FileSystemObject
Set gFileSystemObject = sFSO
End Property

Public Property Get gLogger() As FormattingLogger
Static sLogger As FormattingLogger
If sLogger Is Nothing Then Set sLogger = CreateFormattingLogger("gbd", ProjectName)
Set gLogger = sLogger
End Property

Public Property Get gRegExp() As RegExp
Static sRegexp As RegExp
If sRegexp Is Nothing Then Set sRegexp = New RegExp
Set gRegExp = sRegexp
End Property

'@================================================================================
' Methods
'@================================================================================

Public Function gCreateOutputStream( _
                ByVal pOutputPath As String, _
                ByVal pOutputFilename As String, _
                ByVal pProcessor As IProcessor, _
                ByVal pAppend As Boolean, _
                ByRef pMessage As String) As TextStream
Const ProcName As String = "gCreateOutputStream"
On Error GoTo Err

pOutputPath = gPerformFilenameVariableSubstitution(pOutputPath, pProcessor)
pOutputFilename = gPerformFilenameVariableSubstitution(pOutputFilename, pProcessor)
Dim lFilename As String
lFilename = gFileSystemObject.BuildPath(pOutputPath, pOutputFilename)
Set gCreateOutputStream = CreateWriteableTextFile( _
                                    lFilename, _
                                    pOverwrite:=(Not pAppend), _
                                    pUnicode:=True)

Exit Function

Err:
If Err.Number = 52 Then
    pMessage = "Invalid filename: " & lFilename
ElseIf Err.Number = ErrorCodes.ErrSecurityException Then
    pMessage = "Couldn't create file: " & lFilename & ": " & Err.Description
Else
    gHandleUnexpectedError ProcName, ModuleName
End If
End Function

Public Function gGetContractName(ByVal pContractSpec As IContractSpecifier) As String
AssertArgument Not pContractSpec Is Nothing
gGetContractName = pContractSpec.LocalSymbol & _
                    IIf(pContractSpec.Exchange <> "", "@" & pContractSpec.Exchange, "")
End Function

Public Sub gHandleFatalError(ev As ErrorEventData)
On Error Resume Next    ' ignore any further errors that might arise

gWriteErrorString "Error "
gWriteErrorString CStr(ev.ErrorCode)
gWriteErrorString ": "
gWriteErrorLine ev.ErrorMessage
gWriteErrorLine "At:"
gWriteErrorLine ev.ErrorSource
End Sub

Public Sub gHandleUnexpectedError( _
                ByRef pProcedureName As String, _
                ByRef pModuleName As String, _
                Optional ByRef pFailpoint As String, _
                Optional ByVal pReRaise As Boolean = True, _
                Optional ByVal pLog As Boolean = False, _
                Optional ByVal pErrorNumber As Long, _
                Optional ByRef pErrorDesc As String, _
                Optional ByRef pErrorSource As String)
Dim errSource As String: errSource = IIf(pErrorSource <> "", pErrorSource, Err.Source)
Dim errDesc As String: errDesc = IIf(pErrorDesc <> "", pErrorDesc, Err.Description)
Dim errNum As Long: errNum = IIf(pErrorNumber <> 0, pErrorNumber, Err.Number)

HandleUnexpectedError pProcedureName, ProjectName, pModuleName, pFailpoint, pReRaise, pLog, errNum, errDesc, errSource
End Sub

Public Sub gNotifyUnhandledError( _
                ByRef pProcedureName As String, _
                ByRef pModuleName As String, _
                Optional ByRef pFailpoint As String, _
                Optional ByVal pErrorNumber As Long, _
                Optional ByRef pErrorDesc As String, _
                Optional ByRef pErrorSource As String)
Dim errSource As String: errSource = IIf(pErrorSource <> "", pErrorSource, Err.Source)
Dim errDesc As String: errDesc = IIf(pErrorDesc <> "", pErrorDesc, Err.Description)
Dim errNum As Long: errNum = IIf(pErrorNumber <> 0, pErrorNumber, Err.Number)

UnhandledErrorHandler.Notify pProcedureName, pModuleName, ProjectName, pFailpoint, errNum, errDesc, errSource
End Sub

Public Sub gNotifyAPIConnectionStateChange( _
                ByVal pState As ApiConnectionStates, _
                ByVal pMessage As String)
Const ProcName As String = "gNotifyAPIConnectionStateChange"
On Error GoTo Err

Select Case pState
Case ApiConnNotConnected
    gWriteLineToConsole "Not connected to TWS: " & pMessage
    mProviderReady = False
Case ApiConnConnecting
    gWriteLineToConsole "Connecting to TWS: " & pMessage
    mProviderReady = False
Case ApiConnConnected
    gWriteLineToConsole "Connected to TWS: " & pMessage
    mProviderReady = True
Case ApiConnFailed
    gWriteLineToConsole "Connection to TWS failed: " & pMessage
    mProviderReady = False
End Select

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyDataRetrieved(ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyDataRetrieved"
On Error GoTo Err

LogMessage "Data retrieved from source for contract: " & gGetContractName(pProcessor.ContractSpec)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchCancelled( _
                ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchCancelled"
On Error GoTo Err

If pProcessor.ContractSpec Is Nothing Then
    gWriteLineToConsole "Fetch cancelled for contract " & pProcessor.DataSourceName
Else
    gWriteLineToConsole "Fetch cancelled for contract " & gGetContractName(pProcessor.ContractSpec)
End If
If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchCompleted(ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchCompleted"
On Error GoTo Err

gWriteLineToConsole "Fetch completed: " & _
            pProcessor.NumberOfBarsOutput & _
            " bars for contract: " & _
            gGetContractName(pProcessor.ContractSpec)

If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchFailed( _
                ByVal pProcessor As IProcessor, _
                ByVal pErrorMessage As String)
Const ProcName As String = "gNotifyFetchFailed"
On Error GoTo Err

gWriteLineToConsole "Fetch failed for " & pProcessor.DataSourceName & ": " & pErrorMessage
If mProcessors.Contains(pProcessor) Then mProcessors.Remove pProcessor
Set mCurrentProcessor = Nothing

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyFetchStarted( _
                ByVal pProcessor As IProcessor)
Const ProcName As String = "gNotifyFetchStarted"
On Error GoTo Err

LogMessage "Fetch started for contract " & gGetContractName(pProcessor.ContractSpec)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gNotifyIBServerConnectionClosed()
gWriteLineToConsole "Connection from TWS to IB servers closed"
mProviderReady = False
End Sub

Public Sub gNotifyIBServerConnectionRecovered( _
                ByVal pDataLost As Boolean)
gWriteLineToConsole "Connection from TWS to IB servers recovered"
mProviderReady = True
End Sub

Public Sub gOutputBarToStdOut( _
                ByVal pUseEchoResultFormat As Boolean, _
                ByVal pProcessor As IProcessor, _
                ByVal pBar As Bar, _
                ByVal pTimeframe As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pIncludeMillisecs As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double)
Const ProcName As String = "gOutputBarToStdOut"
On Error GoTo Err

If pBar Is Nothing Then Exit Sub

gWriteLineToStdOut formatBar( _
                        pUseEchoResultFormat, _
                        pProcessor, _
                        pBar, _
                        pBarnumber, _
                        pTimeframe, _
                        pNormaliseDailyBarTimestamps, _
                        pSecType, _
                        pTickSize, _
                        pIncludeMillisecs), _
                    True

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Sub gOutputBarToTextStream( _
                ByVal pProcessor As IProcessor, _
                ByVal pBar As Bar, _
                ByVal pTimeframe As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pIncludeMillisecs As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double, _
                ByVal pStream As TextStream)
Const ProcName As String = "gOutputBarToStdOut"
On Error GoTo Err

If pBar Is Nothing Then Exit Sub

pStream.WriteLine formatBar(False, pProcessor, pBar, pBarnumber, pTimeframe, pNormaliseDailyBarTimestamps, pSecType, pTickSize, pIncludeMillisecs)

Exit Sub

Err:
gHandleUnexpectedError ProcName, ModuleName
End Sub

Public Function gPerformFilenameVariableSubstitution( _
                ByVal pString As String, _
                ByVal pProcessor As IProcessor) As String
Const ProcName As String = "gPerformFilenameVariableSubstitution"
On Error GoTo Err

Dim lContractSpec As IContractSpecifier
Set lContractSpec = pProcessor.ContractSpec

Dim lRegExp As RegExp: Set lRegExp = gRegExp
lRegExp.IgnoreCase = True

lRegExp.Pattern = SubstitutionVariablePattern
lRegExp.Global = True

Dim lMatches As MatchCollection
Set lMatches = lRegExp.Execute(pString)

Dim s As String
Dim lCurrPosn As Long: lCurrPosn = 1

Dim lMatch As Match
For Each lMatch In lMatches
    s = s & Mid$(pString, lCurrPosn, lMatch.FirstIndex - lCurrPosn + 1)
    lCurrPosn = lMatch.FirstIndex + lMatch.Length + 1
    
    Dim r As String
    
    Dim lVariable As String: lVariable = UCase$(lMatch.SubMatches(0))
    Select Case lVariable
    Case ContractVariable
        r = gGetContractName(lContractSpec)
    Case SymbolVariable
        r = lContractSpec.Symbol
    Case LocalSymbolVariable
        r = lContractSpec.LocalSymbol
    Case SecTypeVariable
        r = SecTypeToString(lContractSpec.SecType)
    Case SecTypeAbbrvVariable
        r = SecTypeToShortString(lContractSpec.SecType)
    Case ExchangeVariable
        r = lContractSpec.Exchange
    Case ExpiryVariable
        r = lContractSpec.Expiry
    Case CurrencyVariable
        r = lContractSpec.CurrencyCode
    Case MultiplierVariable
        r = lContractSpec.Multiplier
    Case StrikeVariable
        r = lContractSpec.Strike
    Case RightVariable
        r = OptionRightToString(lContractSpec.Right)
    Case TodayVariable
        r = FormatTimestamp(todayDate, mTimestampDateOnlyFormat)
    Case YesterdayVariable
        r = FormatTimestamp(yesterdayDate, mTimestampDateOnlyFormat)
    Case FromDateVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampDateOnlyFormat)
    Case FromDateTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampFormat + TimestampNoMillisecs)
    Case FromTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
    Case ToDateVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampDateOnlyFormat)
        End If
    Case ToDateTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampFormat + TimestampNoMillisecs)
        End If
    Case ToTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
        End If
    Case TimeframeVariable
        r = pProcessor.Timeframe.ToShortString
    Case NumberRequestedVariable
        r = pProcessor.NumberOfBarsRequested
    Case NumberReturnedVariable
        r = pProcessor.NumberOfBarsOutput
    Case Default
        Assert False, "Unexpected substitution variable: " & lVariable
    End Select
    s = s & escapeNonFilenameChars(r)
Next

gPerformFilenameVariableSubstitution = s & Right$(pString, Len(pString) - lCurrPosn + 1)

Exit Function

Err:
gHandleUnexpectedError ProcName, ModuleName
End Function

Public Function gPerformResultVariableSubstitution( _
                ByVal pProcessor As IProcessor, _
                ByVal pString As String, _
                ByVal pBar As Bar, _
                ByVal pTimePeriod As TimePeriod, _
                ByVal pBarnumber As Long, _
                ByVal pNormaliseDailyBarTimestamps As Boolean, _
                ByVal pSecType As SecurityTypes, _
                ByVal pTickSize As Double, _
                ByVal pIncludeMillisecs As Boolean) As String
Const ProcName As String = "gPerformResultVariableSubstitution"
On Error GoTo Err

Dim lContractSpec As IContractSpecifier
Set lContractSpec = pProcessor.ContractSpec

Dim lRegExp As RegExp: Set lRegExp = gRegExp
lRegExp.IgnoreCase = True

lRegExp.Pattern = SubstitutionVariablePattern
lRegExp.Global = True

Dim lMatches As MatchCollection
Set lMatches = lRegExp.Execute(pString)

Dim s As String
Dim lCurrPosn As Long: lCurrPosn = 1

Dim lMatch As Match
For Each lMatch In lMatches
    s = s & Mid$(pString, lCurrPosn, lMatch.FirstIndex - lCurrPosn + 1)
    lCurrPosn = lMatch.FirstIndex + lMatch.Length + 1
    
    Dim r As String
    
    Dim lVariable As String: lVariable = UCase$(lMatch.SubMatches(0))
    Select Case lVariable
    Case BarNumberVariable
        r = CStr(pBarnumber)
    Case TimestampVariable
        If Not pNormaliseDailyBarTimestamps Then
            r = FormatTimestamp(pBar.TimeStamp, mTimestampFormat Or (Not pIncludeMillisecs And TimestampNoMillisecs))
        ElseIf pTimePeriod.Units = TimePeriodDay Or _
                pTimePeriod.Units = TimePeriodWeek Or _
                pTimePeriod.Units = TimePeriodMonth Or _
                pTimePeriod.Units = TimePeriodYear Then
            r = FormatTimestamp(pBar.TimeStamp, mTimestampDateOnlyFormat)
        Else
            r = FormatTimestamp(pBar.TimeStamp, mTimestampFormat Or (Not pIncludeMillisecs And TimestampNoMillisecs))
        End If
    Case OpenVariable
        r = FormatPrice(pBar.OpenValue, pSecType, pTickSize)
    Case HighVariable
        r = FormatPrice(pBar.HighValue, pSecType, pTickSize)
    Case LowVariable
        r = FormatPrice(pBar.LowValue, pSecType, pTickSize)
    Case CloseVariable
        r = FormatPrice(pBar.CloseValue, pSecType, pTickSize)
    Case VolumeVariable
        r = CStr(pBar.Volume)
    Case TickVolumeVariable
        r = CStr(pBar.TickVolume)
    Case ContractVariable
        r = gGetContractName(lContractSpec)
    Case SymbolVariable
        r = lContractSpec.Symbol
    Case LocalSymbolVariable
        r = lContractSpec.LocalSymbol
    Case SecTypeVariable
        r = SecTypeToString(lContractSpec.SecType)
    Case SecTypeAbbrvVariable
        r = SecTypeToShortString(lContractSpec.SecType)
    Case ExchangeVariable
        r = lContractSpec.Exchange
    Case ExpiryVariable
        r = lContractSpec.Expiry
    Case CurrencyVariable
        r = lContractSpec.CurrencyCode
    Case MultiplierVariable
        r = lContractSpec.Multiplier
    Case StrikeVariable
        r = lContractSpec.Strike
    Case RightVariable
        r = OptionRightToString(lContractSpec.Right)
    Case FromDateVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampDateOnlyFormat)
    Case FromDateTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampFormat + TimestampNoMillisecs)
    Case FromTimeVariable
        r = FormatTimestamp(pProcessor.FromDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
    Case ToDateVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampDateOnlyFormat)
        End If
    Case ToDateTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampFormat + TimestampNoMillisecs)
        End If
    Case ToTimeVariable
        If pProcessor.ToDate = MaxDate Then
            r = LatestParameter
        Else
            r = FormatTimestamp(pProcessor.ToDate, mTimestampTimeOnlyFormat + TimestampNoMillisecs)
        End If
    Case TimeframeVariable
        r = pProcessor.Timeframe.ToShortString
    Case NumberRequestedVariable
        r = pProcessor.NumberOfBarsRequested
    Case NumberReturnedVariable
        r = pProcessor.NumberOfBarsOutput
    Case NewlineVariable
        r = vbCrLf
    Case Default
        Assert False, "Unexpected substitution variable: " & lVariable
    End Select
    s = s & r
Next

gPerformResultVariableSubstitution = s & Right$(pString, Len(pString) - lCurrPosn + 1)

Exit Function

Err:
gHandleUnexpectedError ProcName, ModuleName
End Function

Public Sub gWriteErrorLine( _
                ByVal pMessage As String)
Dim s As String: s = "Line " & mLineNumber & ": " & pMessage
gCon.WriteErrorLine s
mErorStringBuilder.Append s
LogMessage "StdErr: " & mErorStringBuilder.ToString
mErorStringBuilder.Clear
End Sub

Public Sub gWriteErrorString( _
                ByVal pMessage As String)
Dim s As String: s = "Line " & mLineNumber & ": " & pMessage
gCon.WriteErrorString s
mErorStringBuilder.Append s
End Sub

Public Sub gWriteLineToConsole( _
                ByVal pMessage As String)
gCon.WriteLineToConsole pMessage
LogMessage pMessage
End Sub

Public Sub gWriteLineToStdOut( _
                ByVal pMessage As String, _
                Optional ByVal pDontLogit As Boolean)
Const ProcName As String = "gWriteLineToStdOut"

If pMessage = "" Then Exit Sub
If Not pDontLogit Then LogMessage "StdOut: " & pMessage
gCon.WriteLine pMessage
End Sub

Public Sub Main()
On Error GoTo Err

Set mErorStringBuilder = CreateStringBuilder

Set mFatalErrorHandler = New FatalErrorHandler
ApplicationGroupName = "TradeWright"
ApplicationName = "gbd"
SetupDefaultLogging Command

Set gCon = GetConsole

logProgramId

mNumber = &H7FFFFFFF
mTo = MaxDate
mNormaliseDailyBarTimestamps = True

mTimestampFormat = TimestampDateAndTimeISO8601
mTimestampDateOnlyFormat = TimestampDateOnlyISO8601
mTimestampTimeOnlyFormat = TimestampTimeOnlyISO8601

mCommandSeparator = ";"

Set mClp = CreateCommandLineParser(Command)
{'fraction_non_alphanumeric': 0.10095244361829876, 'fraction_numerical': 0.0019081771972824925, 'mean_word_length': 2.5211120764552564, 'pattern_counts': {'":': 10, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 80, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7133212', 'n_tokens_mistral': 14864, 'n_tokens_neox': 14177, 'n_words': 4513}