<commit_before>#!/Users/grant/.virtualenvs/astropy-dev/bin/python3
#!/usr/bin/env python3

"""Generate HTML pages to browse database.

Uses non-standard version of astropy to ensure html anchors retained in
jsviewer tables. This is a virtualenv created into which the modified
version of astropy is installed.

"""

import io
import glob
from os.path import isdir,isfile,basename
from os import mkdir,remove,write,rmdir
import argparse

import numpy as np
from astropy.table import Table,jsviewer
import mysql.connector

from jinja2 import Template
from bokeh.resources import CDN
from bokeh.plotting import figure,output_file,save,ColumnDataSource
import bokeh.palettes
from bokeh.models.widgets import Panel,Tabs
from bokeh.models import HoverTool,OpenURL,TapTool
from bokeh.layouts import gridplot,layout
from bokeh.embed import components

from sdf import templates
from sdf import config as cfg


def cleanup_sample_dirs():
    """Remove dirs for samples that no longer exist."""

    dirs = glob.glob(cfg.www['root']+'samples/*/')
    samp = get_samples()
    for d in dirs:
        dname = basename(d.rstrip('/'))
        if dname not in samp:
            fs = glob.glob(d+'/*')
            fs += glob.glob(d+'/.*')
            print("  {} removed (and files {})".format(d,fs))
            [remove(f) for f in fs]
            rmdir(d)
        else:
            print("  {} ok".format(d))


def get_samples():
    """Get a list of samples.
    
    Get a list of samples from the database. Add "everything" and
    "public" samples, "public" might not show everything in the list,
    but "everything" will (but may not be visible to anyone).
    """
    
    cnx = mysql.connector.connect(user=cfg.mysql['user'],
                                  password=cfg.mysql['<PASSWORD>'],
                                  host=cfg.mysql['host'],
                                  database=cfg.mysql['db_samples'])
    cursor = cnx.cursor(buffered=True)
    cursor.execute("SHOW TABLES;")
    samples = cursor.fetchall() # a list of tuples
    samples = [i[0] for i in samples]
    cursor.close()
    cnx.close()
    return( samples + ['public','everything'] )


def create_dir(wwwroot,sample):
    """Create sample directories and .htaccess if necessary."""

    # make dir and .htaccess if dir doesn't exist
    if not isdir(wwwroot+sample):
        mkdir(wwwroot+sample)

    # make .htaccess if needed, don't put one in "public" or those
    # ending with "_" so stuff in those directories remains visible
    # to those not logged in
    if  sample[-1] != '_' and sample != 'public':
        fd = open(wwwroot+sample+'/.htaccess','w')
        fd.write('AuthName "Must login"\n')
        fd.write('AuthType Basic\n')
        fd.write('AuthUserFile '+cfg.www['root']+'.htpasswd\n')
        fd.write('AuthGroupFile '+cfg.www['root']+'.htgroup\n')
        fd.write('require group admin all '+sample+'\n')
        fd.close()

<commit_msg>"security"<commit_after>#!/Users/grant/.virtualenvs/astropy-dev/bin/python3
#!/usr/bin/env python3

"""Generate HTML pages to browse database.

Uses non-standard version of astropy to ensure html anchors retained in
jsviewer tables. This is a virtualenv created into which the modified
version of astropy is installed.

"""

import io
import glob
from os.path import isdir,isfile,basename
from os import mkdir,remove,write,rmdir
import argparse

import numpy as np
from astropy.table import Table,jsviewer
import mysql.connector

from jinja2 import Template
from bokeh.resources import CDN
from bokeh.plotting import figure,output_file,save,ColumnDataSource
import bokeh.palettes
from bokeh.models.widgets import Panel,Tabs
from bokeh.models import HoverTool,OpenURL,TapTool
from bokeh.layouts import gridplot,layout
from bokeh.embed import components

from sdf import templates
from sdf import config as cfg


def cleanup_sample_dirs():
    """Remove dirs for samples that no longer exist."""

    dirs = glob.glob(cfg.www['root']+'samples/*/')
    samp = get_samples()
    for d in dirs:
        dname = basename(d.rstrip('/'))
        if dname not in samp:
            fs = glob.glob(d+'/*')
            fs += glob.glob(d+'/.*')
            print("  {} removed (and files {})".format(d,fs))
            [remove(f) for f in fs]
            rmdir(d)
        else:
            print("  {} ok".format(d))


def get_samples():
    """Get a list of samples.
    
    Get a list of samples from the database. Add "everything" and
    "public" samples, "public" might not show everything in the list,
    but "everything" will (but may not be visible to anyone).
    """
    
    cnx = mysql.connector.connect(user=cfg.mysql['user'],
                                  password=<PASSWORD>['<PASSWORD>'],
                                  host=cfg.mysql['host'],
                                  database=cfg.mysql['db_samples'])
    cursor = cnx.cursor(buffered=True)
    cursor.execute("SHOW TABLES;")
    samples = cursor.fetchall() # a list of tuples
    samples = [i[0] for i in samples]
    cursor.close()
    cnx.close()
    return( samples + ['public','everything'] )


def create_dir(wwwroot,sample):
    """Create sample directories and .htaccess if necessary."""

    # make dir and .htaccess if dir doesn't exist
    if not isdir(wwwroot+sample):
        mkdir(wwwroot+sample)

    # make .htaccess if needed, don't put one in "public" or those
    # ending with "_" so stuff in those directories remains visible
    # to those not logged in
    if  sample[-1] != '_' and sample != 'public':
        fd = open(wwwroot+sample+'/.htaccess','w')
        fd.write('AuthName "Must login"\n')
        fd.write('AuthType Basic\n')
        fd.write('AuthUserFile '+cfg.www['root']+'.htpasswd\n')
        fd.write('AuthGroupFile '+cfg.www['root']+'.htgroup\n')
        fd.write('require group admin '+sample+'\n')
        fd.close()


{'fraction_non_alphanumeric': 0.12014314928425358, 'fraction_numerical': 0.0017041581458759373, 'mean_word_length': 2.8434839554682383, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3970665', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1748, 'n_words': 549}