Task

Read a file one character at a time, as opposed to reading the entire file at once.

The solution may be implemented as a procedure, which returns the next character in the file on each consecutive call (returning EOF when the end of the file is reached).

The procedure should support the reading of files containing UTF8 encoded wide characters, returning whole characters for each consecutive read.

Related task

   Read a file line by line


constant INVALID_UTF8 = #FFFD
 
function get_one_utf8_char(integer fn)
-- returns INVALID_UTF8 on error, else a string of 1..4 bytes representing one character
object res
integer headb, bytes, c
 
    -- headb = first byte of utf-8 character:
    headb = getc(fn)
    if headb=-1 then return -1 end if
    res = ""&headb
 
    -- calculate length of utf-8 character in bytes (1..4):
    if    headb<0           then bytes = 0  -- (utf-8 starts at #0)
    elsif headb<=0b01111111 then bytes = 1  -- 0b_0xxx_xxxx
    elsif headb<=0b10111111 then bytes = 0  -- (it's a tail byte)
    elsif headb<=0b11011111 then bytes = 2  -- 0b_110x_xxxx
    elsif headb<=0b11101111 then bytes = 3  -- 0b_1110_xxxx
    elsif headb<=0b11110100 then bytes = 4  -- 0b_1111_0xzz
    else                         bytes = 0  -- (utf-8 ends at #10FFFF)
    end if
 
    -- 2..4 bytes encoding (tail range: 0b_1000_0000..0b_1011_1111);
    for j=1 to bytes-1 do                   -- tail bytes are valid?
        c = getc(fn)
        if c<#80 or c>#BF then
            bytes = 0                       -- invalid tail byte or eof
            exit
        end if
        res &= c
    end for
 
    -- 1 byte encoding (head range: 0b_0000_0000..0b_0111_1111):
    if bytes=1 then
        c = headb                               -- UTF-8 = ASCII
 
    -- 2 bytes encoding (head range: 0b_1100_0000..0b_1101_1111):
    elsif bytes=2 then
        c = and_bits(headb, #1F)*#40 +          -- 0b110[7..11] headb
            and_bits(res[2], #3F)               -- 0b10[1..6] tail
        if c>#7FF then ?9/0 end if              -- sanity check
        if c<#80 then                           -- long form?
            res = INVALID_UTF8
        end if
 
    -- 3 bytes encoding (head range: 0b_1110_0000..0b_1110_1111):
    elsif bytes=3 then
        c = and_bits(headb, #0F)*#1000 +        -- 0b1110[13..16] head
            and_bits(res[2], #3F)*#40 +         -- 0b10[7..12] tail
            and_bits(res[3], #3F)               -- 0b10[1..6] tail
        if c>#FFFF then ?9/0 end if             -- sanity check
        if c<#800                               -- long form?
        or (c>=#D800 and c<=#DFFF) then         -- utf-16 incompatible
            res = INVALID_UTF8
        end if
 
    -- 4 bytes encoding (head range: 0b_1111_0000..0b_1111_0111):
    elsif bytes=4 then
        c = and_bits(headb, #07)*#040000 +      -- 0b11110[19..21] head
            and_bits(res[2], #3F)*#1000 +       -- 0b10[13..18] tail
            and_bits(res[3], #3F)*#0040 +       -- 0b10[7..12] tail
            and_bits(res[4], #3F)               -- 0b10[1..6] tail
        if c<#10000                             -- long form?
        or c>#10FFFF then
            res = INVALID_UTF8                  -- utf-8 ends at #10FFFF
        end if
 
    -- bytes = 0; current byte is not encoded correctly:
    else
        res = INVALID_UTF8
    end if
 
    return res
end function
