<commit_before>/*
 * Parse raw binary or hexadecimal data.
 */

export default function parseRaw(data) {
    const contents = removeComments(data);

    const maybeInputType = guessInputType(contents);
    if (maybeInputType.status === "error") {
        return maybeInputType;
    }
    const inputType = maybeInputType.type;

    const answer = extractData(contents, inputType);
    return answer;
}

/*
 * Remove comments, preserving empty lines (for error reporting, etc.).
 */
function removeComments(text) {
    return (text || "").split("\n").map(function(line) {
        // Split on the comment character, and just take the first part
        // (i.e., everything before the comment starts).
        return line.split(";")[0];
    }).join("\n");
};

/*
 * Guess whether this is binary or hex, or fail if neither matches.
 * Returns either { error: <error message string> }
 * or { type: "hex" | "binary" }.
 */
function guessInputType(input) {
    // Make sure the input doesn't have invalid characters.
    const lines = input.split("\n");
    for (let i = 0; i < lines.length; i++) {
        const line = lines[i];
        const match = line.match(/[^\s0-9A-Fa-f]/);
        if (!match) {
            continue;
        }

        const badCharacter = match[0];
        const message =
            `Invalid character "${badCharacter}" at line ${i + 1}`;
        return {
            status: "error",
            message,
        };
    }

    const justData = input.replace(/\s/g, "");
    const isHex = !!justData.match(/[^01]/);

    const shouldDivide = isHex ? 4 : 16;
    if (justData.length % shouldDivide !== 0) {
        const characterNoun = justData.length === 1 ?
            "character" : "characters";
        const dataType = isHex ? "hexadecimal" : "binary";
        const message =
            `Found a total of ${justData.length} ${characterNoun}, ` +
            `but expected length to be a multiple of ${shouldDivide} ` +
            `for ${dataType} data.`;
        return {
            status: "error",
            message,
        };
    }

    return {
        status: "success",
        type: isHex ? "hex" : "binary",
    };
};

function extractData(lines, inputType) {
    const justData = lines.replace(/\s/g, "");

    const charsPerWord = inputType === "hex" ? 4 : 16;
    const words = new Array(justData.length / charsPerWord);
    if (words.length === 0) {
        const message = "Your raw data is empty! " +
            "You need to at least have an origin (.ORIG) address.";
        return {
            status: "error",
            message,
        };<commit_msg>Make the error messages more friendly<commit_after>/*
 * Parse raw binary or hexadecimal data.
 */

export default function parseRaw(data) {
    const contents = removeComments(data);

    const maybeInputType = guessInputType(contents);
    if (maybeInputType.status === "error") {
        return maybeInputType;
    }
    const inputType = maybeInputType.type;

    const answer = extractData(contents, inputType);
    return answer;
}

/*
 * Remove comments, preserving empty lines (for error reporting, etc.).
 */
function removeComments(text) {
    return (text || "").split("\n").map(function(line) {
        // Split on the comment character, and just take the first part
        // (i.e., everything before the comment starts).
        return line.split(";")[0];
    }).join("\n");
};

/*
 * Guess whether this is binary or hex, or fail if neither matches.
 * Returns either { error: <error message string> }
 * or { type: "hex" | "binary" }.
 */
function guessInputType(input) {
    // Make sure the input doesn't have invalid characters.
    const lines = input.split("\n");
    for (let i = 0; i < lines.length; i++) {
        const line = lines[i];
        const match = line.match(/[^\s0-9A-Fa-f]/);
        if (!match) {
            continue;
        }

        const badCharacter = match[0];
        const charCode = badCharacter.charCodeAt(0);
        const message =
            `It looks like you have an invalid character at line ${i + 1}: ` +
            `namely, "${badCharacter}" (code point ${charCode}). ` +
            "Your input should only contain ones and zeros " +
            "(if you're writing binary) " +
            "or digits and letters A through F " +
            "(if you're writing hexadecimal). " +
            "If you're trying to write a comment, " +
            "remember to start it with a semicolon.";
        return {
            status: "error",
            message,
        };
    }

    const justData = input.replace(/\s/g, "");
    const isHex = !!justData.match(/[^01]/);

    const shouldDivide = isHex ? 4 : 16;
    if (justData.length % shouldDivide !== 0) {
        const characterNoun = justData.length === 1 ?
            "character" : "characters";
        const dataType = isHex ? "hexadecimal" : "binary";
        const message =
            `I count a total of ${justData.length} ${characterNoun}, ` +
            `but I expected to find some multiple of ${shouldDivide} ` +
            `for ${dataType} data. ` +
            "You either left out some part of an instruction " +
            "or have a few extras hanging around!";
        return {
            status: "error",
            message,
        };
    }

    return {
        status: "success",
        type: isHex ? "hex" : "binary",
    };
};

function extractData(lines, inputType) {
    const justData = lines.replace(/\s/g, "");

    const charsPerWord = inputType === "hex" ? 4 : 16;
    const words = new Array(justData.length / charsPerWord);
    if (words.length === 0) {
        const message = "Your raw data is empty! " +
            "You need to at least have an origin (.ORIG) address.";
        return {
            status: "error",
            message,
        };
{'fraction_non_alphanumeric': 0.13403303509979353, 'fraction_numerical': 0.006022023399862354, 'mean_word_length': 2.1185622317596566, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6151531', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1630, 'n_words': 532}