<commit_before>/*
 Copyright (C) 2016-2017 <NAME>
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
 Author: <EMAIL> (<NAME>)
 Author: https://github.com/EmielBruijntjes (<NAME>)
*/

#include "myhtml/serialization.h"

/**
 *  Forward declaration of all the functions that are used inside this module
 */
static mystatus_t myhtml_serialization_append(const char* str, size_t size, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_append_attr(const char* str, size_t length, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_attributes(myhtml_tree_t* tree, myhtml_tree_attr_t* attr, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_node_append_text_node(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_node_append_close(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr);

/**
 *  See the function myhtml_serialization_tree_buffer
 */
mystatus_t myhtml_serialization(myhtml_tree_node_t* scope_node, mycore_string_raw_t* str)
{
    return myhtml_serialization_tree_buffer(scope_node, str);
}

/**
 *  See the function myhtml_serialization_node_buffer
 */
mystatus_t myhtml_serialization_node(myhtml_tree_node_t* node, mycore_string_raw_t* str)
{
    return myhtml_serialization_node_buffer(node, str);
}

/**
 *  The serialize function for an entire tree
 *  @param  tree        the tree to be serialized
 *  @param  scope_node  the scope_node 
 *  @param  callback    function that will be called for all strings that have to be printed
 *  @param  ptr         user-supplied pointer
 *  @return bool
 */
mystatus_t myhtml_serialization_tree_callback(myhtml_tree_node_t* scope_node, mycore_callback_serialize_f callback, void *ptr)
{
    myhtml_tree_node_t* node = scope_node;
    
    /*
     Document node it is a root node of tree, it is not a tag element.
     
     For HTML, the root element will always be <HTML> tag.
     It is always present in the HTML document and it always a root of HTML document.
     */
    
    if(scope_node && scope_node->tree && scope_node->tree->document == scope_node) {
        node = scope_node->child;
    }
    
    while(node) {
        if(myhtml_serialization_node_callback(node, callback, ptr))
            return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
        
        if(node->child)
            node = node->child;
        else {
            while(node != scope_node && node->next == NULL) {
                if(myhtml_serialization_node_append_close(node, callback, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                
                node = node->parent;
            }
            
            if(node == scope_node) {
                if(node != node->tree->document) {
                    if(myhtml_serialization_node_append_close(node, callback, ptr))
                        return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                }
                
                break;
            }
            
            if(myhtml_serialization_node_append_close(node, callback, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            node = node->next;
        }
    }
    
    return MyCORE_STATUS_OK;
}

/**
 *  The serialize function for a single node
 *  @param  tree        the tree to be serialized
 *  @param  node        the node that is going to be serialized 
 *  @param  callback    function that will be called for all strings that have to be printed
 *  @param  ptr         user-supplied pointer
 *  @return bool
 */
mystatus_t myhtml_serialization_node_callback(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr)
{
    switch (node->tag_id) {
        case MyHTML_TAG__TEXT: {
            if(myhtml_serialization_node_append_text_node(node, callback, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        case MyHTML_TAG__COMMENT: {
            if(callback("<!--", 4, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token && node->token->str.data) {
                if(callback(node->token->str.data, node->token->str.length, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            }
            
            if(callback("-->", 3, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        case MyHTML_TAG__DOCTYPE: {
            if(callback("<!DOCTYPE", 9, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token) {
                myhtml_tree_attr_t* attr = node->token->attr_first;
                
                if(attr && attr->key.data && attr->key.length) {
                    if(callback(" ", 1, ptr))
                        return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                    
                    if(callback(attr->key.data, attr->key.length, ptr))
                        return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                }
            }
            
            if(callback(">", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        default: {
            size_t length;
            const char *tag = myhtml_tag_name_by_id(node->tree, node->tag_id, &length);

            if(callback("<", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(callback(tag, length, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token) {
                if(myhtml_serialization_attributes(node->tree, node->token->attr_first, callback, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            }
            
            if(callback(">", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
    }<commit_msg>fix doctype serialization<commit_after>/*
 Copyright (C) 2016-2017 <NAME>
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public
 License as published by the Free Software Foundation; either
 version 2.1 of the License, or (at your option) any later version.
 
 This library is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 Lesser General Public License for more details.
 
 You should have received a copy of the GNU Lesser General Public
 License along with this library; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
 Author: <EMAIL> (<NAME>)
 Author: https://github.com/EmielBruijntjes (<NAME>)
*/

#include "myhtml/serialization.h"

/**
 *  Forward declaration of all the functions that are used inside this module
 */
static mystatus_t myhtml_serialization_append(const char* str, size_t size, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_append_attr(const char* str, size_t length, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_attributes(myhtml_tree_t* tree, myhtml_tree_attr_t* attr, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_node_append_text_node(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr);
static mystatus_t myhtml_serialization_node_append_close(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr);

/**
 *  See the function myhtml_serialization_tree_buffer
 */
mystatus_t myhtml_serialization(myhtml_tree_node_t* scope_node, mycore_string_raw_t* str)
{
    return myhtml_serialization_tree_buffer(scope_node, str);
}

/**
 *  See the function myhtml_serialization_node_buffer
 */
mystatus_t myhtml_serialization_node(myhtml_tree_node_t* node, mycore_string_raw_t* str)
{
    return myhtml_serialization_node_buffer(node, str);
}

/**
 *  The serialize function for an entire tree
 *  @param  tree        the tree to be serialized
 *  @param  scope_node  the scope_node 
 *  @param  callback    function that will be called for all strings that have to be printed
 *  @param  ptr         user-supplied pointer
 *  @return bool
 */
mystatus_t myhtml_serialization_tree_callback(myhtml_tree_node_t* scope_node, mycore_callback_serialize_f callback, void *ptr)
{
    myhtml_tree_node_t* node = scope_node;
    
    /*
     Document node it is a root node of tree, it is not a tag element.
     
     For HTML, the root element will always be <HTML> tag.
     It is always present in the HTML document and it always a root of HTML document.
     */
    
    if(scope_node && scope_node->tree && scope_node->tree->document == scope_node) {
        node = scope_node->child;
    }
    
    while(node) {
        if(myhtml_serialization_node_callback(node, callback, ptr))
            return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
        
        if(node->child)
            node = node->child;
        else {
            while(node != scope_node && node->next == NULL) {
                if(myhtml_serialization_node_append_close(node, callback, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                
                node = node->parent;
            }
            
            if(node == scope_node) {
                if(node != node->tree->document) {
                    if(myhtml_serialization_node_append_close(node, callback, ptr))
                        return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                }
                
                break;
            }
            
            if(myhtml_serialization_node_append_close(node, callback, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            node = node->next;
        }
    }
    
    return MyCORE_STATUS_OK;
}

/**
 *  The serialize function for a single node
 *  @param  tree        the tree to be serialized
 *  @param  node        the node that is going to be serialized 
 *  @param  callback    function that will be called for all strings that have to be printed
 *  @param  ptr         user-supplied pointer
 *  @return bool
 */
mystatus_t myhtml_serialization_node_callback(myhtml_tree_node_t* node, mycore_callback_serialize_f callback, void *ptr)
{
    switch (node->tag_id) {
        case MyHTML_TAG__TEXT: {
            if(myhtml_serialization_node_append_text_node(node, callback, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        case MyHTML_TAG__COMMENT: {
            if(callback("<!--", 4, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token && node->token->str.data) {
                if(callback(node->token->str.data, node->token->str.length, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            }
            
            if(callback("-->", 3, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        case MyHTML_TAG__DOCTYPE: {
            if(callback("<!DOCTYPE", 9, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(callback(" ", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token) {
                myhtml_tree_attr_t* attr = node->token->attr_first;
                
                if(attr && attr->key.data && attr->key.length) {
                    if(callback(attr->key.data, attr->key.length, ptr))
                        return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
                }
            }
            
            if(callback(">", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
        default: {
            size_t length;
            const char *tag = myhtml_tag_name_by_id(node->tree, node->tag_id, &length);

            if(callback("<", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(callback(tag, length, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            if(node->token) {
                if(myhtml_serialization_attributes(node->tree, node->token->attr_first, callback, ptr))
                    return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            }
            
            if(callback(">", 1, ptr))
                return MyCORE_STATUS_ERROR_MEMORY_ALLOCATION;
            
            break;
        }
    }
{'fraction_non_alphanumeric': 0.0754689115344853, 'fraction_numerical': 0.004135282823807414, 'mean_word_length': 2.305589455699292, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 83, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5298827', 'n_tokens_mistral': 3902, 'n_tokens_neox': 3753, 'n_words': 988}