<commit_before>"""
Various Helpers

"""

import docutils.core
from docutils.parsers.rst import directives, Directive

import os
import pygments
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, Number
import subprocess
from docutils_html5 import Writer
from flask import g
from traceback import format_exc
import base64
import pygal

ROOT = os.path.join(os.path.dirname(__file__), '..', 'projects')


class PygmentsStyle(Style):
    """Pygments style based on Solarized."""
    styles = {
        Keyword: '#b58900',
        Name: '#cb4b16',
        Comment: '#839496',
        String: '#2aa198',
        Error: '#dc322f',
        Number: '#859900'}


class Pygments(Directive):
    """Code syntax hightlighting."""
    required_arguments = 1
    optional_arguments = 1
    final_argument_whitespace = True
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        lexer_name = self.arguments[1] if len(self.arguments) > 1 else 'python'
        code = open(filename).read()
        lexer = get_lexer_by_name(lexer_name)
        formatter = HtmlFormatter(
            noclasses=True, nobackground=True, style=PygmentsStyle)
        parsed = pygments.highlight(code, lexer, formatter)
        return [docutils.nodes.raw('', parsed, format='html')]


class InlinePygments(Directive):
    """Inline code sytax highlighting."""
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = True
    has_content = True

    def run(self):
        lexer = get_lexer_by_name(self.arguments[0]
                                  if len(self.arguments) == 1 else 'python')
        code = '\n'.join(self.content)
        formatter = HtmlFormatter(
            noclasses=True, nobackground=True, style=PygmentsStyle)
        parsed = pygments.highlight(code, lexer, formatter)
        return [docutils.nodes.raw('', parsed, format='html')]


class UrlGet(Directive):
    """Serve the response of a request."""
    required_arguments = 2
    optional_arguments = 0
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        url = self.arguments[1]
        parts = filename.split('/')
        cwd = '/'.join(parts[:-1])
        pipe = subprocess.Popen(
            ['python', filename, url], cwd=cwd, stdout=subprocess.PIPE)
        content = pipe.communicate()[0].strip()
        retcode = pipe.poll()
        if retcode:
            content = 'An error occured %s' % '-'.join([filename, url])
        content = content.replace(
            '<html', '<html style="background-color: white"')
        content = '<iframe src="data:text/html;base64,%s">' \
            'Request Output</iframe>' % base64.b64encode(content)
        # Remove EOLs
        content = content.replace('\n', '')
        return [docutils.nodes.raw('', content, format='html')]


class PyResult(Directive):
    """Execute the given python file and puts its result in the document."""
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        pipe = subprocess.Popen(['python', filename], stdout=subprocess.PIPE)
        content = pipe.communicate()[0]
        retcode = pipe.poll()
        if retcode:
            content = 'An error occured'
        content = '<pre class="script-output">%s</pre>' % content
        return [docutils.nodes.raw('', content, format='html')]


class Pygal(Directive):
    """Execute the given python file and puts its result in the document."""
    required_arguments = 0
    optional_arguments = 2
    final_argument_whitespace = True
    has_content = True

    def run(self):
        width, height = map(int, self.arguments[:2]) if len(
            self.arguments) >= 2 else (600, 400)
        if len(self.arguments) == 1:
            self.render_fix = bool(self.arguments[0])
        elif len(self.arguments) == 3:
            self.render_fix = bool(self.arguments[2])
        else:
            self.render_fix = False
        self.content = list(self.content)
        content = list(self.content)
        if self.render_fix:
            content[-1] = 'rv = ' + content[-1]
        code = '\n'.join(content)
        locals = {'pygal': pygal}
        try:
            exec(code, {}, locals)
        except Exception:
            return [docutils.nodes.system_message(
                'An exception as occured during code parsing:'
                ' \n %s' % format_exc(),
                level=3)]
        if self.render_fix:
            rv = locals['rv']
        else:
            chart = None
            for key, value in locals.items():
                if isinstance(value, pygal.ghost.Ghost):
                    chart = value
                    self.content.append(key + '.render()')
                    break
            if chart is None:
                return [docutils.nodes.system_message(
                    'No instance of graph found', level=3)]
            chart.config.width = width
            chart.config.height = height<commit_msg>Fix python 3 exec scope problem<commit_after>"""
Various Helpers

"""

import docutils.core
from docutils.parsers.rst import directives, Directive

import os
import pygments
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter
from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, Number
import subprocess
from docutils_html5 import Writer
from flask import g
from traceback import format_exc
import base64
import pygal

ROOT = os.path.join(os.path.dirname(__file__), '..', 'projects')


class PygmentsStyle(Style):
    """Pygments style based on Solarized."""
    styles = {
        Keyword: '#b58900',
        Name: '#cb4b16',
        Comment: '#839496',
        String: '#2aa198',
        Error: '#dc322f',
        Number: '#859900'}


class Pygments(Directive):
    """Code syntax hightlighting."""
    required_arguments = 1
    optional_arguments = 1
    final_argument_whitespace = True
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        lexer_name = self.arguments[1] if len(self.arguments) > 1 else 'python'
        code = open(filename).read()
        lexer = get_lexer_by_name(lexer_name)
        formatter = HtmlFormatter(
            noclasses=True, nobackground=True, style=PygmentsStyle)
        parsed = pygments.highlight(code, lexer, formatter)
        return [docutils.nodes.raw('', parsed, format='html')]


class InlinePygments(Directive):
    """Inline code sytax highlighting."""
    required_arguments = 0
    optional_arguments = 1
    final_argument_whitespace = True
    has_content = True

    def run(self):
        lexer = get_lexer_by_name(self.arguments[0]
                                  if len(self.arguments) == 1 else 'python')
        code = '\n'.join(self.content)
        formatter = HtmlFormatter(
            noclasses=True, nobackground=True, style=PygmentsStyle)
        parsed = pygments.highlight(code, lexer, formatter)
        return [docutils.nodes.raw('', parsed, format='html')]


class UrlGet(Directive):
    """Serve the response of a request."""
    required_arguments = 2
    optional_arguments = 0
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        url = self.arguments[1]
        parts = filename.split('/')
        cwd = '/'.join(parts[:-1])
        pipe = subprocess.Popen(
            ['python', filename, url], cwd=cwd, stdout=subprocess.PIPE)
        content = pipe.communicate()[0].strip()
        retcode = pipe.poll()
        if retcode:
            content = 'An error occured %s' % '-'.join([filename, url])
        content = content.replace(
            '<html', '<html style="background-color: white"')
        content = '<iframe src="data:text/html;base64,%s">' \
            'Request Output</iframe>' % base64.b64encode(content)
        # Remove EOLs
        content = content.replace('\n', '')
        return [docutils.nodes.raw('', content, format='html')]


class PyResult(Directive):
    """Execute the given python file and puts its result in the document."""
    required_arguments = 1
    optional_arguments = 0
    final_argument_whitespace = True
    has_content = False

    def run(self):
        filename = os.path.join(ROOT, g.project_name, self.arguments[0])
        pipe = subprocess.Popen(['python', filename], stdout=subprocess.PIPE)
        content = pipe.communicate()[0]
        retcode = pipe.poll()
        if retcode:
            content = 'An error occured'
        content = '<pre class="script-output">%s</pre>' % content
        return [docutils.nodes.raw('', content, format='html')]


class Pygal(Directive):
    """Execute the given python file and puts its result in the document."""
    required_arguments = 0
    optional_arguments = 2
    final_argument_whitespace = True
    has_content = True

    def run(self):
        width, height = map(int, self.arguments[:2]) if len(
            self.arguments) >= 2 else (600, 400)
        if len(self.arguments) == 1:
            self.render_fix = bool(self.arguments[0])
        elif len(self.arguments) == 3:
            self.render_fix = bool(self.arguments[2])
        else:
            self.render_fix = False
        self.content = list(self.content)
        content = list(self.content)
        if self.render_fix:
            content[-1] = 'rv = ' + content[-1]
        code = '\n'.join(content)
        scope = {'pygal': pygal}
        try:
            exec(code, scope)
        except Exception:
            return [docutils.nodes.system_message(
                'An exception as occured during code parsing:'
                ' \n %s' % format_exc(),
                level=3)]
        if self.render_fix:
            rv = scope['rv']
        else:
            chart = None
            for key, value in scope.items():
                if isinstance(value, pygal.ghost.Ghost):
                    chart = value
                    self.content.append(key + '.render()')
                    break
            if chart is None:
                return [docutils.nodes.system_message(
                    'No instance of graph found', level=3)]
            chart.config.width = width
            chart.config.height = height
{'fraction_non_alphanumeric': 0.1180961357210179, 'fraction_numerical': 0.013854853911404336, 'mean_word_length': 2.3803759158967823, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '873385', 'n_tokens_mistral': 3210, 'n_tokens_neox': 3029, 'n_words': 782}