<commit_before>#!/usr/bin/env python3

'''
This script generates an html file with a tree visualization, using d3.js.
The template (template_visualize_rst_tree.html) is based off some D3.js
examples:
http://mbostock.github.io/d3/talk/20111018/tree.html
http://bl.ocks.org/mbostock/4063570
http://www.d3noob.org/2013/01/adding-tooltips-to-d3js-graph.html

For the D3.js license, see LICENSE_d3.txt or
https://github.com/mbostock/d3/blob/master/LICENSE for the D3.js license.

Note: on very large trees, edges may cross.  This appears to be a feature of
the layout algorithm.  It could be avoided by having curved edges, but I don't
think functionality for computing the necessary paths is available.
'''

import argparse
import json
import os

from nltk.tree import ParentedTree
from jinja2 import Environment, FileSystemLoader


THIS_FILE_DIRNAME = os.path.dirname(os.path.abspath(__file__))


def convert_tree_json(input_json):
    '''
    convert the JSON from the RST parser into a format for D3.js.
    '''
    tree = ParentedTree.fromstring(input_json["scored_rst_trees"][0]["tree"])
    edus = [' '.join(x) for x in input_json["edu_tokens"]]

    res = convert_tree_json_helper(tree, edus)

    return res


def convert_tree_json_helper(subtree, edus):
    if subtree.label() == 'text':<commit_msg>added note about extracting SVG files from the visualization output<commit_after>#!/usr/bin/env python3

'''
This script generates an html file with a tree visualization, using d3.js.
The template (template_visualize_rst_tree.html) is based off some D3.js
examples:
http://mbostock.github.io/d3/talk/20111018/tree.html
http://bl.ocks.org/mbostock/4063570
http://www.d3noob.org/2013/01/adding-tooltips-to-d3js-graph.html

For the D3.js license, see LICENSE_d3.txt or
https://github.com/mbostock/d3/blob/master/LICENSE for the D3.js license.

If you want to extract an SVG file from the output HTML file, use
this: http://nytimes.github.io/svg-crowbar/.

Note: On very large trees, edges may cross.  This appears to be a feature of
the layout algorithm.  It could be avoided by having curved edges, but I don't
think functionality for computing the necessary paths is available.
'''

import argparse
import json
import os

from nltk.tree import ParentedTree
from jinja2 import Environment, FileSystemLoader


THIS_FILE_DIRNAME = os.path.dirname(os.path.abspath(__file__))


def convert_tree_json(input_json):
    '''
    convert the JSON from the RST parser into a format for D3.js.
    '''
    tree = ParentedTree.fromstring(input_json["scored_rst_trees"][0]["tree"])
    edus = [' '.join(x) for x in input_json["edu_tokens"]]

    res = convert_tree_json_helper(tree, edus)

    return res


def convert_tree_json_helper(subtree, edus):
    if subtree.label() == 'text':
{'fraction_non_alphanumeric': 0.10273972602739725, 'fraction_numerical': 0.024513338139870222, 'mean_word_length': 4.367504835589942, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3190729', 'n_tokens_mistral': 969, 'n_tokens_neox': 906, 'n_words': 276}