<commit_before>import json
import subprocess
import os
import logging

import numpy as np
import requests
import time

logger = logging.getLogger(__name__)

_host = None #: PlotHost
DEFAULT_SERVER = 'http://localhost:2908'


def plotvr(data, col=None, size=None, type='p', lines=None, label=None,
           name=None, description=None, speed=None, auto_scale=True,
           digits=5, host=None, return_data=False, push_data=True):
    # TODO assert compatibility checks
    n = data.shape[0]
    for i in [col, size, lines, label]:
        assert i is None or i.shape == (n,), f"Parameters need to have same length: {i} has shape {i.shape} but would need {(n,)}"
    if auto_scale:
        # have all variables scaled to [-1,1]
        ranges = data.max(0) - data.min(0)
        ranges[ranges == 0] = 1
        data = (data - data.min(0)) / ranges * 2 - 1
    if col is None:
        payload = data[:,:3]
    else:
        payload = np.hstack((data[:,:3],col.reshape((-1,1))))
    # todo: remove NAs, center and scale...
    body = {'data': payload.tolist(),'speed': 0, 'protocolVersion': '0.3.0'}
    if col is not None: body['col'] = col.tolist()
    if size is not None: body['size'] = size.tolist()
    if type is not None: body['type'] = type
    if label is not None: body['label'] = label.tolist()
    if speed is not None: body['speed'] = speed
    metadata = { 'n': n, 'created': time.ctime() }
    metadata['name'] = name or "Dataset"
    if description is not None: metadata['description'] = description
    body['metadata'] = metadata
    # data_json = json.dumps(, allow_nan=False)
    if push_data:
        plot_host = get_host(host)
        plot_host.post(json=body)
    if return_data:
        return body

<commit_msg>added surfacevr<commit_after>import json
import subprocess
import os
import logging

import numpy as np
import requests
import time

logger = logging.getLogger(__name__)

_host = None #: PlotHost
DEFAULT_SERVER = 'http://localhost:2908'


def plotvr(data, col=None, size=None, type='p', lines=None, label=None,
           name=None, description=None, speed=None, auto_scale=True,
           digits=5, host=None, return_data=False, push_data=True):
    # TODO assert compatibility checks
    n = data.shape[0]
    for i in [col, size, lines, label]:
        assert i is None or i.shape == (n,), f"Parameters need to have same length: {i} has shape {i.shape} but would need {(n,)}"
    if auto_scale:
        # have all variables scaled to [-1,1]
        ranges = data.max(0) - data.min(0)
        ranges[ranges == 0] = 1
        data = (data - data.min(0)) / ranges * 2 - 1
    if col is None:
        payload = data[:,:3]
    else:
        payload = np.hstack((data[:,:3],col.reshape((-1,1))))
    # todo: remove NAs, center and scale...
    body = {'data': payload.tolist(),'speed': 0, 'protocolVersion': '0.3.0'}
    if col is not None: body['col'] = col.tolist()
    if size is not None: body['size'] = size.tolist()
    if type is not None: body['type'] = type
    if label is not None: body['label'] = label.tolist()
    if speed is not None: body['speed'] = speed
    metadata = { 'n': n, 'created': time.ctime() }
    metadata['name'] = name or "Dataset"
    if description is not None: metadata['description'] = description
    body['metadata'] = metadata
    # data_json = json.dumps(, allow_nan=False)
    if push_data:
        plot_host = get_host(host)
        plot_host.post(json=body)
    if return_data:
        return body

def surfacevr(data, col=None,
           name=None, description=None, speed=None, autoScale=True,
           digits=5, host="http://localhost:2908", returnData=False):
    global _host
    _host = host
    # TODO assert compatibility checks
    n,m = data.shape
    for i in [col]:
        assert i is None or i.shape == data.shape, f"Parameters need to have same shape: {i} has shape {i.shape} but would need {data.shape}"
    # todo: remove NAs, center and scale...
    body = {'surface': {'data':data.tolist(), 'col':col},'speed': 0, 'protocolVersion': '0.3.0'}
    if speed is not None: body['speed'] = speed
    metadata = { 'n': n, 'm': m, 'created': time.ctime() }
    metadata['name'] = name or "Dataset"
    if description is not None: metadata['description'] = description
    body['metadata'] = metadata
    # data_json = json.dumps(, allow_nan=False)
    if host is not None:
        requests.post(host, json=body)
    if returnData:
        return body

{'fraction_non_alphanumeric': 0.14311431143114312, 'fraction_numerical': 0.012376237623762377, 'mean_word_length': 3.0482695810564664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2981657', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1431, 'n_words': 456}