<commit_before>#lang racket
(require "registry.rkt" "common.rkt")
(provide parse-tree unparse-tree tree-copy tree=?
         node? node-type node-parent node-children
         node-get node-get* node-set! node-set*! node-add! node-remove!
         node-prev node-next node-fchild node-lchild in-tree)

(struct node (type attrs parent children) #:mutable
        #:methods gen:custom-write
        [(define (write-proc node port mode)
           (fprintf port "[~a ~a]" (node-type node) (string-join (map ~a (dict->attributes (node-attrs node))) " ")))])

(define (tree-copy tree)
  (assert (not (node-parent tree)) "To (tree-copy) you must pass the root of the tree")
  (parse-tree (unparse-tree tree)))

(define (tree=? tree1 tree2)
  (equal? (unparse-tree tree1) (unparse-tree tree2)))

(define (parse-tree tree)
  (let loop ([tree tree] [parent #f])
    (match-define `([,type ,attrs ...] ,subtrees ...) tree)
    (define n (node type (attributes->dict attrs) parent (void)))
    (set-node-children! n (map (curryr loop n) (filter list? subtrees)))
    n))

(define (unparse-tree node)
  (list*
   (list* (node-type node) (dict->attributes (node-attrs node)))
   (map unparse-tree (node-children node))))

(define (node-get node name #:default [default #f])
  (apply values (node-get* node name #:default (list default))))

(define (node-get* node name #:default [default #f])
  (dict-ref (node-attrs node) name default))

(define (node-set! node name value)
  (set-node-attrs! node (dict-set (node-attrs node) name (list value))))

(define (node-add! node name value)
  (set-node-attrs! node (cons value (dict-ref (node-attrs node) name '()))))

(define (node-set*! node name value)
  (set-node-attrs! node (dict-set (node-attrs node) name value)))

(define (node-remove! node name)
  (set-node-attrs! node (dict-remove (node-attrs node) name)))

(define (node-prev node)
  (define sibs (if (node-parent node) (node-children (node-parent node)) (list node)))
  (for/first ([prev (sequence-cons #f sibs)] [node* sibs] #:when (equal? node node*)) prev))

(define (node-next node)
  (define sibs (if (node-parent node) (node-children (node-parent node)) (list node)))
  (match (member node sibs) [(list _ next _ ...) next] [(list _) #f]))

(define (node-fchild node)
  (if (null? (node-children node)) #f (first (node-children node))))

(define (node-lchild node)
  (if (null? (node-children node)) #f (last (node-children node))))

(define (in-tree node)
  (apply sequence-append (in-value node) (map in-tree (node-children node))))
<commit_msg>Remember strings in HTML trees<commit_after>#lang racket
(require "registry.rkt" "common.rkt")
(provide parse-tree unparse-tree tree-copy tree=?
         node? node-type node-parent node-children* node-children node-attrs
         node-get node-get* node-set! node-set*! node-add! node-remove!
         node-prev node-next node-fchild node-lchild in-tree)

(struct node (type attrs parent children*) #:mutable
        #:methods gen:custom-write
        [(define (write-proc node port mode)
           (fprintf port "[~a ~a]" (node-type node) (string-join (map ~a (dict->attributes (node-attrs node))) " ")))])

(define (node-children node)
  (filter node? (node-children* node)))

(define (tree-copy tree)
  (assert (not (node-parent tree)) "To (tree-copy) you must pass the root of the tree")
  (parse-tree (unparse-tree tree)))

(define (tree=? tree1 tree2)
  (equal? (unparse-tree tree1) (unparse-tree tree2)))

(define (parse-tree tree)
  (let loop ([tree tree] [parent #f])
    (match-define `([,type ,attrs ...] ,subtrees ...) tree)
    (define n (node type (attributes->dict attrs) parent (void)))
    (set-node-children*! n (for/list ([k subtrees]) (if (list? k) (loop k n) k)))
    n))

(define (unparse-tree node)
  (if (node? node)
      (list*
       (list* (node-type node) (dict->attributes (node-attrs node)))
       (map unparse-tree (node-children* node)))
      node))

(define (node-get node name #:default [default #f])
  (apply values (node-get* node name #:default (list default))))

(define (node-get* node name #:default [default #f])
  (dict-ref (node-attrs node) name default))

(define (node-set! node name value)
  (set-node-attrs! node (dict-set (node-attrs node) name (list value))))

(define (node-add! node name value)
  (set-node-attrs! node (cons value (dict-ref (node-attrs node) name '()))))

(define (node-set*! node name value)
  (set-node-attrs! node (dict-set (node-attrs node) name value)))

(define (node-remove! node name)
  (set-node-attrs! node (dict-remove (node-attrs node) name)))

(define (node-prev node)
  (define sibs (if (node-parent node) (node-children (node-parent node)) (list node)))
  (for/first ([prev (sequence-cons #f sibs)] [node* sibs] #:when (equal? node node*)) prev))

(define (node-next node)
  (define sibs (if (node-parent node) (node-children (node-parent node)) (list node)))
  (match (member node sibs) [(list _ next _ ...) next] [(list _) #f]))

(define (node-fchild node)
  (if (null? (node-children node)) #f (first (node-children node))))

(define (node-lchild node)
  (if (null? (node-children node)) #f (last (node-children node))))

(define (in-tree node)
  (apply sequence-append (in-value node) (map in-tree (node-children node))))

{'fraction_non_alphanumeric': 0.17487169739593234, 'fraction_numerical': 0.001520623455616803, 'mean_word_length': 3.8677150786308974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1297831', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1788, 'n_words': 582}