<commit_before>' Extracts entire text from Ichitaro document as utf-8 text file.
' Works only on Windows with Justsystem Ichitaro installed.
' 
' usage> CScript ichitaro-to-text.vbs /normalize IchitaroFile DesDir
' 
' https://github.com/sukuba/js-py-document-search

' On Error Resume Next
Set Args = WScript.Arguments
Main Args.Named, Args.Unnamed(0), Args.Unnamed(1)
' If Err.Number <> 0 Then WScript.Echo Err.Description
WScript.Quit(Err.Number)

Sub Main(Opts, Src, Dest)
  If Opts.Exists("nojxw") Then
    texts = ForDebug()
  Else
    Set texts = GetIchitaroText(Src)
  End If
  
  If Opts.Exists("dump") Then
    DumpTexts texts
  End If
  
  DestFileName = MergePath(Src, Dest)
  WScript.Echo Src, DestFileName
  
  If Opts.Exists("ansi") Then
    SaveTextFile texts, DestFileName
  Else
    SaveUtf8TextFile texts, DestFileName
  End If
  
  If Opts.Exists("normalize") Then
    RunCmd "normalize-text.bat " & DestFileName
  End If
End Sub

Function GetIchitaroText(FileName)
  Set app = CreateObject("JXW.Application")
  app.Visible = True
  app.Document.Open FileName
  app.TaroLibrary.SelectAll 1
  Set texts = app.TaroLibrary.GetString()
  app.TaroLibrary.QuitDocumentWindow
  app.Quit
  Set app = Nothing
  
  Set GetIchitaroText = texts
End Function

Sub DumpTexts(texts)
  For Each text in texts
    WScript.Echo text
  Next
End Sub
<commit_msg>make ichitaro work<commit_after>' Extracts entire text from Ichitaro document as utf-8 text file.
' Works only on Windows with Justsystem Ichitaro installed.
' 
' usage> CScript ichitaro-to-text.vbs /normalize IchitaroFile DesDir
' 
' https://github.com/sukuba/js-py-document-search

' On Error Resume Next
Set Args = WScript.Arguments
Main Args.Named, Args.Unnamed(0), Args.Unnamed(1)
' If Err.Number <> 0 Then WScript.Echo Err.Description
WScript.Quit(Err.Number)

' it's not nice, but uses global variable to keep the application between open and quit.
' because texts object is an Ichitaro object, it disappears when the app quit.
Dim app

Sub Main(Opts, Src, Dest)
  If Opts.Exists("nojxw") Then
    texts = ForDebug()
  Else
    Set texts = GetIchitaroText(Src)
  End If
  
  If Opts.Exists("dump") Then
    DumpTexts texts
  End If
  
  DestFileName = MergePath(Src, Dest)
  WScript.Echo Src, DestFileName
  
  If Opts.Exists("ansi") Then
    SaveTextFile texts, DestFileName
  Else
    SaveUtf8TextFile texts, DestFileName
  End If
  
  If Not Opts.Exists("nojxw") Then
    Set texts = Nothing
    QuitIchitaro
  End If
  
  If Opts.Exists("normalize") Then
    RunCmd "normalize-text.bat " & DestFileName
  End If
End Sub

Function GetIchitaroText(FileName)
  Set app = CreateObject("JXW.Application")
  app.Visible = True
  app.Documents.Open FileName
  app.TaroLibrary.SelectAll 1
  Set texts = app.TaroLibrary.GetString()
  app.TaroLibrary.QuitDocumentWindow
  'app.Quit
  'Set app = Nothing
  
  Set GetIchitaroText = texts
End Function

Sub QuitIchitaro()
  app.Quit
  Set app = Nothing
End Sub

Sub DumpTexts(texts)
  For Each text in texts
    WScript.Echo text
  Next
End Sub

{'fraction_non_alphanumeric': 0.08349705304518663, 'fraction_numerical': 0.003929273084479371, 'mean_word_length': 3.6712538226299696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5354769', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1063, 'n_words': 310}