<commit_before>import six
import os
import glob
import copy
import json
if six.PY2:  # python 2
    from Queue import Queue
    from ConfigParser import NoOptionError
else:  # python 3
    from queue import Queue
    from configparser import NoOptionError
import shutil
import random
import functools
import binascii
import mimetypes
import threading
import warnings
import webbrowser
import numpy as np
from tornado import web

from .FallbackLoader import FallbackLoader
from .. import utils, options, volume, dataset
from ..database import db
from . import serve
from .data import Package

try:
    cmapdir = options.config.get('webgl', 'colormaps')
    if not os.path.exists(cmapdir):
        raise Exception("Colormap directory (%s) does not exits"%cmapdir)
except NoOptionError:
    cmapdir = os.path.join(options.config.get("basic", "filestore"), "colormaps")
    if not os.path.exists(cmapdir):
        raise Exception("Colormap directory was not defined in the config file and the default (%s) does not exits"%cmapdir)


colormaps = glob.glob(os.path.join(cmapdir, "*.png"))
colormaps = [(os.path.splitext(os.path.split(cm)[1])[0], serve.make_base64(cm))
             for cm in sorted(colormaps)]

def make_static(outpath, data, types=("inflated",), recache=False, cmap="RdBu_r",
                template="static.html", layout=None, anonymize=False, overlays_available=None,
                html_embed=True, overlays_visible=('rois', 'sulci'), labels_visible=('rois', ),
                overlay_file=None, copy_ctmfiles=True, title='Brain', **kwargs):
    """
    Creates a static webGL MRI viewer in your filesystem so that it can easily
    be posted publically for sharing or just saved for later viewing.

    Parameters
    ----------
    outpath : string
        The directory where the static viewer will be saved. Will be created if it
        doesn't already exist.
    data : Dataset object or implicit Dataset
        Dataset object containing all the data you wish to plot. Can be any type
        of implicit dataset, such as a single Volume, Vertex, etc. object or a
        dictionary of Volume, Vertex. etc. objects.
    recache : bool, optional
        Force recreation of CTM and SVG files for surfaces. Default False
    template : string, optional
        Name of template HTML file. Default 'static.html'
    anonymize : bool, optional
        Whether to rename CTM and SVG files generically, for public distribution.
        Default False
    overlays_available : tuple, optional
        Overlays availble in the viewer. If None, then all overlay layers of the
        svg file will be potentially available in the viewer (whether initially
        visible or not). This provides the option to include, e.g., only a subset
        of layers for a given static viewer.
    overlays_visible : tuple, optional
        The listed overlay layers will be set visible by default. Layers not listed
        here will be hidden by default (but can be enabled in the viewer GUI).
        Default ('rois', 'sulci')
    labels_visible : tuple, optional
        Labels for the listed layers will be set visible by default. Labels for
        layers not listed here will be hidden by default (but can be enabled in
        the viewer GUI). Default ('rois', )
    **kwargs
        All additional keyword arguments are passed to the template renderer.

    Other parameters
    ----------------
    types : tuple, optional
        Types of surfaces to include in addition to the original (fiducial, pial,
        and white matter) and flat surfaces. Default ('inflated', )
    cmap : string, optional
        Name of default colormap. Default 'RdBu_r'
        TODO: DOES THIS DO ANYTHING ANYMORE?
    overlay_file : str, optional
        Custom overlays.svg file to use instead of the default one for this
        subject (if not None). Default None.
    html_embed : bool, optional
        Whether to embed the webgl resources in the html output.  Default 'True'.
        If 'False', the webgl resources must be served by your web server.
    copy_ctmfiles : bool, optional
        Whether to copy the CTM files to the static directory.  Default 'True'.
        In some use cases, the same CTM data will be used in many static views. To
        avoid duplication of files, set to 'False'.  (The datastore cache must
        then be served with your web server).
    title : str, optional
        The title that is displayed on the viewer website when it is loaded in
        a browser.

    Notes
    -----
    You will need a real web server to view this, since `file://` paths
    don't handle xsrf correctly
    """

    outpath = os.path.abspath(os.path.expanduser(outpath)) # To handle ~ expansion
    if not os.path.exists(outpath):
        os.makedirs(outpath)
    if not os.path.exists(os.path.join(outpath, 'data')):
        # Don't lump together w/ outpath, because of edge cases
        # for which outpath exists but not sub-folder `data`
        os.makedirs(os.path.join(outpath, "data"))

    data = dataset.normalize(data)
    if not isinstance(data, dataset.Dataset):
        data = dataset.Dataset(data=data)

    db.auxfile = data

    package = Package(data)
    subjects = list(package.subjects)

    ctmargs = dict(method='mg2', level=9, recache=recache, external_svg=overlay_file,
                   overlays_available=overlays_available)
    ctms = dict((subj, utils.get_ctmpack(subj, types, **ctmargs))
                for subj in subjects)
    package.reorder(ctms)

    db.auxfile = None
    if layout is None:
        layout = [None, (1, 1), (2, 1), (3, 1), (2, 2), (3, 2), (3, 2), (3, 3), (3, 3), (3, 3)][len(subjects)]

    ## Rename files to anonymize
    submap = dict()
    print(list(ctms.keys()))
    for i, (subj, ctmfile) in enumerate(ctms.items()):
        print("Iterating through ctms, on # %d"%i)
        print(subj)
        oldpath, fname = os.path.split(ctmfile)
        fname, ext = os.path.splitext(fname)
        if anonymize:
            newfname = "S%d"%i
            submap[subj] = newfname
            # del ctms[subj]
            # subj = newfname
            # print("--> changed to: %s"%subj)
        else:
            newfname = fname
        ctms[subj] = newfname+".json"

        for ext in ['json', 'ctm', 'svg']:
            srcfile = os.path.join(oldpath, "%s.%s"%(fname, ext))
            newfile = os.path.join(outpath, "%s.%s"%(newfname, ext))
            if os.path.exists(newfile):
                os.unlink(newfile)

            if os.path.exists(srcfile) and copy_ctmfiles:
                shutil.copy2(srcfile, newfile)

            if ext == "json" and anonymize:
                ## change filenames in json
                nfh = open(newfile)
                jsoncontents = nfh.read()
                nfh.close()

                ofh = open(newfile, "w")
                ofh.write(jsoncontents.replace(fname, newfname))
                ofh.close()
    if anonymize:
        old_subjects = sorted(list(ctms.keys()))<commit_msg>removal of extraneous debugging print statements<commit_after>import six
import os
import glob
import copy
import json
if six.PY2:  # python 2
    from Queue import Queue
    from ConfigParser import NoOptionError
else:  # python 3
    from queue import Queue
    from configparser import NoOptionError
import shutil
import random
import functools
import binascii
import mimetypes
import threading
import warnings
import webbrowser
import numpy as np
from tornado import web

from .FallbackLoader import FallbackLoader
from .. import utils, options, volume, dataset
from ..database import db
from . import serve
from .data import Package

try:
    cmapdir = options.config.get('webgl', 'colormaps')
    if not os.path.exists(cmapdir):
        raise Exception("Colormap directory (%s) does not exits"%cmapdir)
except NoOptionError:
    cmapdir = os.path.join(options.config.get("basic", "filestore"), "colormaps")
    if not os.path.exists(cmapdir):
        raise Exception("Colormap directory was not defined in the config file and the default (%s) does not exits"%cmapdir)


colormaps = glob.glob(os.path.join(cmapdir, "*.png"))
colormaps = [(os.path.splitext(os.path.split(cm)[1])[0], serve.make_base64(cm))
             for cm in sorted(colormaps)]

def make_static(outpath, data, types=("inflated",), recache=False, cmap="RdBu_r",
                template="static.html", layout=None, anonymize=False, overlays_available=None,
                html_embed=True, overlays_visible=('rois', 'sulci'), labels_visible=('rois', ),
                overlay_file=None, copy_ctmfiles=True, title='Brain', **kwargs):
    """
    Creates a static webGL MRI viewer in your filesystem so that it can easily
    be posted publically for sharing or just saved for later viewing.

    Parameters
    ----------
    outpath : string
        The directory where the static viewer will be saved. Will be created if it
        doesn't already exist.
    data : Dataset object or implicit Dataset
        Dataset object containing all the data you wish to plot. Can be any type
        of implicit dataset, such as a single Volume, Vertex, etc. object or a
        dictionary of Volume, Vertex. etc. objects.
    recache : bool, optional
        Force recreation of CTM and SVG files for surfaces. Default False
    template : string, optional
        Name of template HTML file. Default 'static.html'
    anonymize : bool, optional
        Whether to rename CTM and SVG files generically, for public distribution.
        Default False
    overlays_available : tuple, optional
        Overlays availble in the viewer. If None, then all overlay layers of the
        svg file will be potentially available in the viewer (whether initially
        visible or not). This provides the option to include, e.g., only a subset
        of layers for a given static viewer.
    overlays_visible : tuple, optional
        The listed overlay layers will be set visible by default. Layers not listed
        here will be hidden by default (but can be enabled in the viewer GUI).
        Default ('rois', 'sulci')
    labels_visible : tuple, optional
        Labels for the listed layers will be set visible by default. Labels for
        layers not listed here will be hidden by default (but can be enabled in
        the viewer GUI). Default ('rois', )
    **kwargs
        All additional keyword arguments are passed to the template renderer.

    Other parameters
    ----------------
    types : tuple, optional
        Types of surfaces to include in addition to the original (fiducial, pial,
        and white matter) and flat surfaces. Default ('inflated', )
    cmap : string, optional
        Name of default colormap. Default 'RdBu_r'
        TODO: DOES THIS DO ANYTHING ANYMORE?
    overlay_file : str, optional
        Custom overlays.svg file to use instead of the default one for this
        subject (if not None). Default None.
    html_embed : bool, optional
        Whether to embed the webgl resources in the html output.  Default 'True'.
        If 'False', the webgl resources must be served by your web server.
    copy_ctmfiles : bool, optional
        Whether to copy the CTM files to the static directory.  Default 'True'.
        In some use cases, the same CTM data will be used in many static views. To
        avoid duplication of files, set to 'False'.  (The datastore cache must
        then be served with your web server).
    title : str, optional
        The title that is displayed on the viewer website when it is loaded in
        a browser.

    Notes
    -----
    You will need a real web server to view this, since `file://` paths
    don't handle xsrf correctly
    """

    outpath = os.path.abspath(os.path.expanduser(outpath)) # To handle ~ expansion
    if not os.path.exists(outpath):
        os.makedirs(outpath)
    if not os.path.exists(os.path.join(outpath, 'data')):
        # Don't lump together w/ outpath, because of edge cases
        # for which outpath exists but not sub-folder `data`
        os.makedirs(os.path.join(outpath, "data"))

    data = dataset.normalize(data)
    if not isinstance(data, dataset.Dataset):
        data = dataset.Dataset(data=data)

    db.auxfile = data

    package = Package(data)
    subjects = list(package.subjects)

    ctmargs = dict(method='mg2', level=9, recache=recache, external_svg=overlay_file,
                   overlays_available=overlays_available)
    ctms = dict((subj, utils.get_ctmpack(subj, types, **ctmargs))
                for subj in subjects)
    package.reorder(ctms)

    db.auxfile = None
    if layout is None:
        layout = [None, (1, 1), (2, 1), (3, 1), (2, 2), (3, 2), (3, 2), (3, 3), (3, 3), (3, 3)][len(subjects)]

    ## Rename files to anonymize
    submap = dict()
    for i, (subj, ctmfile) in enumerate(ctms.items()):
        oldpath, fname = os.path.split(ctmfile)
        fname, ext = os.path.splitext(fname)
        if anonymize:
            newfname = "S%d"%i
            submap[subj] = newfname
        else:
            newfname = fname
        ctms[subj] = newfname+".json"

        for ext in ['json', 'ctm', 'svg']:
            srcfile = os.path.join(oldpath, "%s.%s"%(fname, ext))
            newfile = os.path.join(outpath, "%s.%s"%(newfname, ext))
            if os.path.exists(newfile):
                os.unlink(newfile)

            if os.path.exists(srcfile) and copy_ctmfiles:
                shutil.copy2(srcfile, newfile)

            if ext == "json" and anonymize:
                ## change filenames in json
                nfh = open(newfile)
                jsoncontents = nfh.read()
                nfh.close()

                ofh = open(newfile, "w")
                ofh.write(jsoncontents.replace(fname, newfname))
                ofh.close()
    if anonymize:
        old_subjects = sorted(list(ctms.keys()))
{'fraction_non_alphanumeric': 0.09251101321585903, 'fraction_numerical': 0.004044197299053947, 'mean_word_length': 2.5681525380056685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '193126', 'n_tokens_mistral': 4100, 'n_tokens_neox': 3946, 'n_words': 1487}