<commit_before>from __future__ import unicode_literals
import uuid, json
from pandas import DataFrame
from mpcontribs.config import mp_level01_titles, mp_id_pattern, object_id_pattern
from recdict import RecursiveDict
from utils import disable_ipython_scrollbar
from IPython.display import display_html, display, HTML, Image

class Tree(RecursiveDict):
    """class to hold and display single tree of hierarchical data"""
    def __init__(self, content):
        super(Tree, self).__init__(
            (key, value) for key, value in content.iteritems()
            if key not in mp_level01_titles[2:] and \
            not key.startswith(mp_level01_titles[1])
        )

class HierarchicalData(RecursiveDict):
    """class to hold and display all hierarchical data in MPFile"""
    def __init__(self, document):
        super(HierarchicalData, self).__init__(
            (identifier, Tree(content))
            for identifier, content in document.iteritems()
        )

    @property
    def general(self):
        return self[mp_level01_titles[0]]

    def __str__(self):
        return 'mp-ids: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        display_html('<h2>Hierarchical Data</h2>', raw=True)
        for identifier, hdata in self.iteritems():
            if identifier != mp_level01_titles[0]:
                display_html('<h3>{}</h3>'.format(identifier), raw=True)
            display_html(hdata)

def get_backgrid_table(df):
    """Backgrid-conform dict from DataFrame"""
    # shorten global import times by importing django here
    import numpy as np
    from django.core.validators import URLValidator
    from django.core.exceptions import ValidationError
    from pandas import MultiIndex
    val = URLValidator()
    table = dict()
    nrows = df.shape[0]
    nrows_max = 200
    if nrows > nrows_max:
        df = df.head(n=nrows_max)
    numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()

    if isinstance(df.index, MultiIndex):
        df.reset_index(inplace=True)

    table['columns'] = []
    table['rows'] = super(Table, df).to_dict(orient='records')

    for col_index, col in enumerate(df.columns):
        cell_type = 'number'

        # avoid looping rows to minimize use of `df.iat` (time-consuming in 3d)
        if not col.startswith('level_') and col not in numeric_columns:
            is_url_column, prev_unit = True, None

            for row_index in xrange(nrows):
                cell = unicode(df.iat[row_index, col_index])
                cell_split = cell.split(' ', 1)
                if not cell or len(cell_split) == 1: # empty cell or no space
                    if is_url_column:
                        is_url_column = bool(not cell or mp_id_pattern.match(cell))
                        if is_url_column:
                            if cell:
                                value = 'https://materialsproject.org/materials/{}'.format(cell)
                                table['rows'][row_index][col] = value
                        else:
                            try:
                                val(cell)
                                is_url_column = True
                            except ValidationError:
                                # is_url_column already set to False
                                break
                else:
                    value, unit = cell_split # TODO convert cell_split[0] to float?
                    if prev_unit is None:
                        is_url_column = False
                        prev_unit = unit
                    elif prev_unit != unit:
                        break

            cell_type = 'uri' if is_url_column else 'string'

        col_split = col.split('##')
        nesting = [col_split[0]] if len(col_split) > 1 else []
        table['columns'].append({
            'name': col,  'cell': cell_type, 'nesting': nesting, 'editable': 0
        })
        if len(col_split) > 1:
            table['columns'][-1].update({'label': '##'.join(col_split[1:])})
        if len(table['columns']) > 9:
            table['columns'][-1]['renderable'] = 0

    return table

def render_dataframe(df, webapp=False):
    """use BackGrid JS library to render Pandas DataFrame"""
    # TODO check for index column in df other than the default numbering
    uuid_str, uuid_str_paginator = str(uuid.uuid4()), str(uuid.uuid4())
    uuid_str_filter = str(uuid.uuid4())
    table = get_backgrid_table(df)
    table_str = json.dumps(table)
    html = "<div id='{}'></div>".format(uuid_str_filter)
    html += "<div id='{}' style='width:100%;'></div>".format(uuid_str)
    html += "<div id='{}'></div>".format(uuid_str_paginator)
    html += "<script>"
    if webapp:
        html += "requirejs(['main'], function() {"
    html += """<commit_msg>backgrid_table: detect unit and add to col name<commit_after>from __future__ import unicode_literals
import uuid, json
from pandas import DataFrame
from mpcontribs.config import mp_level01_titles, mp_id_pattern, object_id_pattern
from recdict import RecursiveDict
from utils import disable_ipython_scrollbar
from IPython.display import display_html, display, HTML, Image

class Tree(RecursiveDict):
    """class to hold and display single tree of hierarchical data"""
    def __init__(self, content):
        super(Tree, self).__init__(
            (key, value) for key, value in content.iteritems()
            if key not in mp_level01_titles[2:] and \
            not key.startswith(mp_level01_titles[1])
        )

class HierarchicalData(RecursiveDict):
    """class to hold and display all hierarchical data in MPFile"""
    def __init__(self, document):
        super(HierarchicalData, self).__init__(
            (identifier, Tree(content))
            for identifier, content in document.iteritems()
        )

    @property
    def general(self):
        return self[mp_level01_titles[0]]

    def __str__(self):
        return 'mp-ids: {}'.format(' '.join(self.keys()))

    def _ipython_display_(self):
        display_html('<h2>Hierarchical Data</h2>', raw=True)
        for identifier, hdata in self.iteritems():
            if identifier != mp_level01_titles[0]:
                display_html('<h3>{}</h3>'.format(identifier), raw=True)
            display_html(hdata)

def get_backgrid_table(df):
    """Backgrid-conform dict from DataFrame"""
    # shorten global import times by importing django here
    import numpy as np
    from django.core.validators import URLValidator
    from django.core.exceptions import ValidationError
    from pandas import MultiIndex
    val = URLValidator()
    table = dict()
    nrows = df.shape[0]
    nrows_max = 200
    if nrows > nrows_max:
        df = df.head(n=nrows_max)
    numeric_columns = df.select_dtypes(include=[np.number]).columns.tolist()

    if isinstance(df.index, MultiIndex):
        df.reset_index(inplace=True)

    table['columns'] = []
    table['rows'] = super(Table, df).to_dict(orient='records')

    for col_index, col in enumerate(list(df.columns)):
        cell_type = 'number'

        # avoid looping rows to minimize use of `df.iat` (time-consuming in 3d)
        if not col.startswith('level_') and col not in numeric_columns:
            is_url_column, prev_unit, old_col = True, None, col

            for row_index in xrange(nrows):
                cell = unicode(df.iat[row_index, col_index])
                cell_split = cell.split(' ', 1)
                if not cell or len(cell_split) == 1: # empty cell or no space
                    if is_url_column:
                        is_url_column = bool(not cell or mp_id_pattern.match(cell))
                        if is_url_column:
                            if cell:
                                value = 'https://materialsproject.org/materials/{}'.format(cell)
                                table['rows'][row_index][col] = value
                        else:
                            try:
                                val(cell)
                                is_url_column = True
                            except ValidationError:
                                # is_url_column already set to False
                                break
                else:
                    value, unit = cell_split # TODO convert cell_split[0] to float?
                    table['rows'][row_index].pop(old_col)
                    if prev_unit is None:
                        is_url_column = False
                        prev_unit = unit
                        col = '{} [{}]'.format(col, unit)
                    table['rows'][row_index][col] = cell if prev_unit != unit else value

            cell_type = 'uri' if is_url_column else 'string'

        col_split = col.split('##')
        nesting = [col_split[0]] if len(col_split) > 1 else []
        table['columns'].append({
            'name': col, 'cell': cell_type, 'nesting': nesting, 'editable': 0
        })
        if len(col_split) > 1:
            table['columns'][-1].update({'label': '##'.join(col_split[1:])})
        if len(table['columns']) > 9:
            table['columns'][-1]['renderable'] = 0

    return table

def render_dataframe(df, webapp=False):
    """use BackGrid JS library to render Pandas DataFrame"""
    # TODO check for index column in df other than the default numbering
    uuid_str, uuid_str_paginator = str(uuid.uuid4()), str(uuid.uuid4())
    uuid_str_filter = str(uuid.uuid4())
    table = get_backgrid_table(df)
    table_str = json.dumps(table)
    html = "<div id='{}'></div>".format(uuid_str_filter)
    html += "<div id='{}' style='width:100%;'></div>".format(uuid_str)
    html += "<div id='{}'></div>".format(uuid_str_paginator)
    html += "<script>"
    if webapp:
        html += "requirejs(['main'], function() {"
    html += """
{'fraction_non_alphanumeric': 0.11257668711656442, 'fraction_numerical': 0.008384458077709612, 'mean_word_length': 2.1602584814216477, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '906153', 'n_tokens_mistral': 2896, 'n_tokens_neox': 2769, 'n_words': 724}