<commit_before>
import ../frontend/[BuildParams, CommandLine]
import text/Buffer
import io/[FileReader, File]
import ../middle/Module

/* Will go into the load method of Token */
nullToken : Token
nullToken = Token new(0, 0, null)

Token: cover {
    
    start, length : SizeT
    module: Module
    
    new: static func~fromData (data: Int*, module: Module) -> This {
        this : This
        this start =  data[0]
        this length = data[1]
        this module = module
        return this
    }
    
    new: static func (.start, .length, .module) -> This {
        this : This
        this start =  start
        this length = length
        this module = module
        return this
    }
    
    new: static func~copy (origin: This) -> This {
        // well that's quite stupid. but covers have value semantics
        // already, so no action is needed to make a "copy" of it.
        return origin
    }
    
    toString: func -> String {
        module != null ? (
            "%s [%d, %d]" format(module getFullName(), getStart(), getEnd())
        ) : (
            "[%d, %d]" format(getStart(), getEnd())
        )
    }
    
    throwWarning: func (message: String) {
        printMessage(message, "[WARNING]")
    }
    
    throwError: func (message: String) {
        printMessage(message, "[ERROR]")
        if(BuildParams fatalError) CommandLine failure()
    }
    
    printMessage: func ~noPrefix (message, type: String) {
        printMessage("", message, type)
    }
        
    printMessage: func (prefix, message, type: String) {
        if(module == null) {
            Exception new(This, "From unknown source [%s] %s" format(type, message)) throw()
        }
        
        fr := FileReader new(getPath())
        
        lastNewLine := 0
        lines := 1
        idx := 0
        // zap the lines before we start
        while(fr hasNext() && idx < start) {
            c := fr read()
            if(c == '\n') {
                lines += 1
                lastNewLine = idx
            }
            idx += 1
        }
        
        // zap the end of the line that contains us
        while(true) {
            if(!fr hasNext() || fr read() == '\n') break
            idx += 1
        }
        
        fr reset(lastNewLine == 0 ? 0 : lastNewLine + 1)
        over := Buffer new()
        
        if(type != "") {
            prefix print()
            "%s:%d:%d %s %s" format(module path + ".ooc", lines, start - lastNewLine, type, message) println()
        } else if(message != "") {
            prefix print()
            message println()
        }
        
        prefix print()
        end := getEnd()
        for(i in (lastNewLine + 1)..(idx + 1)) {
            c := fr read()
            if(c == '\t') {
                printf("    ")
                over append("    ")
            } else {
                printf("%c", c)
                if(i < start || i >= end) {
                    over append(' ')
                } else {
                    over append('^')
                }
            }
        }
        prefix print()
        over toString() println()<commit_msg>Fixed bug where newlines would be needed but not displayed in error messages involving .ooc files extracts<commit_after>
import ../frontend/[BuildParams, CommandLine]
import text/Buffer
import io/[FileReader, File]
import ../middle/Module

/* Will go into the load method of Token */
nullToken : Token
nullToken = Token new(0, 0, null)

Token: cover {
    
    start, length : SizeT
    module: Module
    
    new: static func~fromData (data: Int*, module: Module) -> This {
        this : This
        this start =  data[0]
        this length = data[1]
        this module = module
        return this
    }
    
    new: static func (.start, .length, .module) -> This {
        this : This
        this start =  start
        this length = length
        this module = module
        return this
    }
    
    new: static func~copy (origin: This) -> This {
        // well that's quite stupid. but covers have value semantics
        // already, so no action is needed to make a "copy" of it.
        return origin
    }
    
    toString: func -> String {
        module != null ? (
            "%s [%d, %d]" format(module getFullName(), getStart(), getEnd())
        ) : (
            "[%d, %d]" format(getStart(), getEnd())
        )
    }
    
    throwWarning: func (message: String) {
        printMessage(message, "[WARNING]")
    }
    
    throwError: func (message: String) {
        printMessage(message, "[ERROR]")
        if(BuildParams fatalError) CommandLine failure()
    }
    
    printMessage: func ~noPrefix (message, type: String) {
        printMessage("", message, type)
    }
        
    printMessage: func (prefix, message, type: String) {
        if(module == null) {
            Exception new(This, "From unknown source [%s] %s" format(type, message)) throw()
        }
        
        fr := FileReader new(getPath())
        
        lastNewLine := 0
        lines := 1
        idx := 0
        // zap the lines before we start
        while(fr hasNext() && idx < start) {
            c := fr read()
            if(c == '\n') {
                lines += 1
                lastNewLine = idx
            }
            idx += 1
        }
        
        // zap the end of the line that contains us
        while(true) {
            if(!fr hasNext() || fr read() == '\n') break
            idx += 1
        }
        
        fr reset(lastNewLine == 0 ? 0 : lastNewLine + 1)
        over := Buffer new()
        
        if(type != "") {
            prefix print()
            "%s:%d:%d %s %s" format(module path + ".ooc", lines, start - lastNewLine, type, message) println()
        } else if(message != "") {
            prefix print()
            message println()
        }
        
        prefix print()
        end := getEnd()
        for(i in (lastNewLine + 1)..(idx + 1)) {
            c := fr read()
            match (c) {
                case '\t' =>
                    printf("    ")
                    over append("    ")
                case '\n' =>
                    break // the outer loop, not the match.
                case =>
                    printf("%c", c)
                    if(i < start || i >= end) {
                        over append(' ')
                    } else {
                        over append('^')
                    }
            }
        }
        println()
        prefix print()
        over toString() println()
{'fraction_non_alphanumeric': 0.12304866850321396, 'fraction_numerical': 0.004591368227731864, 'mean_word_length': 1.3772280829392507, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2244385', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1775, 'n_words': 575}