<commit_before>// STM32 microprocessor firmware updater agent
// Copyright (c) 2014 Electric Imp
// This file is licensed under the MIT License
// http://opensource.org/licenses/MIT

// GLOBALS AND CONSTS ----------------------------------------------------------
const BLOCKSIZE = 4096; // size of binary image chunks to send down to device
const DELAY_DONE_NOTIFICATION_BY = 0.1; // pause to let the device write the final chunk before ending download

enum filetypes {
    INTELHEX,
    BIN
}

hex_buffer <- "";
bin_buffer <- blob(2);
fetch_url <- "";
fw_ptr <- 0;
fw_len <- 0;
filetype <- null;
bin_ptr <- 0;
bin_len <- 0;
bytes_sent <- 0;

// FUNCTION AND CLASS DEFINITIONS ----------------------------------------------

// Helper: Parses a hex string and turns it into an integer
// Input: hexidecimal number as a string
// Return: number (integer)
function hextoint(str) {
    local hex = 0x0000;
    foreach (ch in str) {
        local nibble;
        if (ch >= '0' && ch <= '9') {
            nibble = (ch - '0');
        } else {
            nibble = (ch - 'A' + 10);
        }
        hex = (hex << 4) + nibble;
    }
    return hex;
}

// Helper: Parse a buffer of hex into the bin_buffer
// Input: Intel Hex data (string)
// Return: None
//      (writes binary into bin_buffer)
function parse_hexfile(hex) {
    try {
        // line up are start at the point in the bin buffer we've already parsed up to
        bin_buffer.seek(bin_len);
        // make sure we have enough bin buffer to write in all the things we're going to parse
        // for (local i = bin_len; i < BLOCKSIZE; i++) bin_buffer.writen(0xFF, 'b');
        // bin_buffer.seek(bin_len);
        
        local from = 0, to = 0, line = "", offset = 0x00000000;
        do {
            if (to < 0 || to == null || to >= hex.len()) break;
            from = hex.find(":", to);
            
            if (from < 0 || from == null || from + 1 >= hex.len()) break;
            to = hex.find(":", from + 1);
            
            if (to < 0 || to == null || from >= to || to >= hex.len()) break;
            line = hex.slice(from + 1, to);
            //server.log(format("[%d,%d] => %s", from, to, line));
            
            if (line.len() > 10) {
                local len = hextoint(line.slice(0, 2));
                local addr = hextoint(line.slice(2, 6));
                local type = hextoint(line.slice(6, 8));
 
                // Ignore all record types except 00, which is a data record. 
                // Look out for 02 records which set the high order byte of the address space
                if (type == 0) {
                    // Normal data record
                //} else if (type == 4 && len == 2 && addr == 0 && line.len() > 12) {
                } else if (type == 4) {
                    //server.log(format("Type 4 Line, len %d, addr %08x, line len %d", len, addr, line.len()));
                    // Set the offset
                    offset = hextoint(line.slice(8, 12)); // << 16;
                    if (offset != 0) {
                        //server.log(format("Set offset to 0x%08X", offset));
                        //server.log(format("From: %d, To: %d",from,to));
                        // right now, we ignore offset changes and assume the full images will be contiguous!
                    }
                    continue;
                } else {
                    //server.log("Skipped: " + line)
                    continue;
                }
 
                // Read the data from 8 to the end (less the last checksum byte)
                for (local i = 8; i < (8 + (len * 2)); i += 2) {
                    local datum = hextoint(line.slice(i, i + 2));
                    bin_buffer.writen(datum, 'b')
                }
                
                // Checking the checksum would be a good idea but skipped for now
                local checksum = hextoint(line.slice(-2));
            }
        } while (from != null && to != null && from < to);
        
        // Resize the raw hex buffer so that it starts at the next line we need to parse
        //server.log(format("Resizing Hex Buffer [%d to %d]",from,hex_buffer.len()));
        hex_buffer = hex_buffer.slice(from, hex_buffer.len());
        
        bin_len += (bin_buffer.tell() - bin_len);
        bin_buffer.seek(bin_ptr);
        //server.log(format("Done parsing chunk, %d bytes in bin buffer",bin_len));
        
        //server.log("Free RAM: " + (imp.getmemoryfree()/1024) + " kb")
        return true;
        
    } catch (e) {
        server.log(e)
        return false;
    }
}

function finish_dl() {
    device.send("dl_complete",0);
    server.log("Download complete");
    fetch_url = "";
    fw_ptr = 0;
    fw_len = 0;
    filetype = null;
    // reclaim memory
    hex_buffer = "";<commit_msg>added line-by-line checksum verification for intel hex<commit_after>// STM32 microprocessor firmware updater agent
// Copyright (c) 2014 Electric Imp
// This file is licensed under the MIT License
// http://opensource.org/licenses/MIT

// GLOBALS AND CONSTS ----------------------------------------------------------
const BLOCKSIZE = 4096; // size of binary image chunks to send down to device
const DELAY_DONE_NOTIFICATION_BY = 0.1; // pause to let the device write the final chunk before ending download

enum filetypes {
    INTELHEX,
    BIN
}

hex_buffer <- "";
bin_buffer <- blob(2);
fetch_url <- "";
fw_ptr <- 0;
fw_len <- 0;
filetype <- null;
bin_ptr <- 0;
bin_len <- 0;
bytes_sent <- 0;

// FUNCTION AND CLASS DEFINITIONS ----------------------------------------------

// Helper: Parses a hex string and turns it into an integer
// Input: hexidecimal number as a string
// Return: number (integer)
function hextoint(str) {
    local hex = 0x0000;
    foreach (ch in str) {
        local nibble;
        if (ch >= '0' && ch <= '9') {
            nibble = (ch - '0');
        } else {
            nibble = (ch - 'A' + 10);
        }
        hex = (hex << 4) + nibble;
    }
    return hex;
}

// Helper: Compute the 1-byte modular sum of a line of Intel Hex
// Input: Line of Intel Hex (string)
// Return: modular sum (integer, 1-byte)
function modularsum(line) {
    local sum = 0x00
    for (local i = 0; i < line.len(); i+=2) {
        sum += hextoint(line.slice(i,i + 2));
    }
    return ((~sum + 1) & 0xff);
}

// Helper: Parse a buffer of hex into the bin_buffer
// Input: Intel Hex data (string)
// Return: None
//      (writes binary into bin_buffer)
function parse_hexfile(hex) {
    try {
        // line up are start at the point in the bin buffer we've already parsed up to
        bin_buffer.seek(bin_len);
        // make sure we have enough bin buffer to write in all the things we're going to parse
        // for (local i = bin_len; i < BLOCKSIZE; i++) bin_buffer.writen(0xFF, 'b');
        // bin_buffer.seek(bin_len);
        
        local from = 0, to = 0, line = "", offset = 0x00000000;
        do {
            if (to < 0 || to == null || to >= hex.len()) break;
            from = hex.find(":", to);
            
            if (from < 0 || from == null || from + 1 >= hex.len()) break;
            to = hex.find(":", from + 1);
            
            if (to < 0 || to == null || from >= to || to >= hex.len()) break;
            // make sure to strip nasty trailing \r\n
            line = rstrip(hex.slice(from + 1, to));
            //server.log(format("[%d,%d] => %s", from, to, line));
            
            if (line.len() > 10) {
                local len = hextoint(line.slice(0, 2));
                local addr = hextoint(line.slice(2, 6));
                local type = hextoint(line.slice(6, 8));
 
                // Ignore all record types except 00, which is a data record. 
                // Look out for 02 records which set the high order byte of the address space
                if (type == 0) {
                    // Normal data record
                //} else if (type == 4 && len == 2 && addr == 0 && line.len() > 12) {
                } else if (type == 4) {
                    //server.log(format("Type 4 Line, len %d, addr %08x, line len %d", len, addr, line.len()));
                    // Set the offset
                    offset = hextoint(line.slice(8, 12)); // << 16;
                    if (offset != 0) {
                        //server.log(format("Set offset to 0x%08X", offset));
                        //server.log(format("From: %d, To: %d",from,to));
                        // right now, we ignore offset changes and assume the full images will be contiguous!
                    }
                    continue;
                } else {
                    //server.log("Skipped: " + line)
                    continue;
                }
 
                // Read the data from 8 to the end (less the last checksum byte)
                for (local i = 8; i < (8 + (len * 2)); i += 2) {
                    local datum = hextoint(line.slice(i, i + 2));
                    bin_buffer.writen(datum, 'b')
                }
                
                // Checking the checksum would be a good idea but skipped for now
                local read = hextoint(line.slice(line.len() - 2, line.len()));
                local calc = modularsum(line.slice(0,line.len() - 2));
                if (read != calc) {
                    throw format("Hex File Checksum Error: %02x (read) != %02x (calc) [%s]", read, calc, line);
                }
            }
        } while (from != null && to != null && from < to);
        
        // Resize the raw hex buffer so that it starts at the next line we need to parse
        //server.log(format("Resizing Hex Buffer [%d to %d]",from,hex_buffer.len()));
        hex_buffer = hex_buffer.slice(from, hex_buffer.len());
        
        bin_len += (bin_buffer.tell() - bin_len);
        bin_buffer.seek(bin_ptr);
        //server.log(format("Done parsing chunk, %d bytes in bin buffer",bin_len));
        
        //server.log("Free RAM: " + (imp.getmemoryfree()/1024) + " kb")
        return true;
        
    } catch (e) {
        server.log(e)
        return false;
    }
}

function finish_dl() {
    device.send("dl_complete",0);
    server.log("Download complete");
    fetch_url = "";
    fw_ptr = 0;
    fw_len = 0;
    filetype = null;
    // reclaim memory
    hex_buffer = "";
{'fraction_non_alphanumeric': 0.1517675544794189, 'fraction_numerical': 0.01937046004842615, 'mean_word_length': 1.5572065378900446, 'pattern_counts': {'":': 4, '<': 44, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1199631', 'n_tokens_mistral': 3261, 'n_tokens_neox': 2972, 'n_words': 1021}