<commit_before>import config, json, cgi, sys, Websheet, re, os
from utils import *

cfg = config.config_jo
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa364232(v=vs.85).aspx
blacklisted_words_txt = """
using 
StreamReader
StreamWriter
File
.Read
FileStream
OpenText
System.IO

The following functions are used with file I/O.

    CancelIo 
    CancelIoEx 
    CancelSynchronousIo 
    CreateIoCompletionPort 
    FlushFileBuffers 
    GetQueuedCompletionStatus 
    GetQueuedCompletionStatusEx 
    LockFile 
    LockFileEx 
    PostQueuedCompletionStatus 
    ReadFile 
    ReadFileEx 
    ReadFileScatter 
    SetEndOfFile 
    SetFileCompletionNotificationModes 
    SetFileIoOverlappedRange 
    SetFilePointer 
    SetFilePointerEx 
    UnlockFile 
    UnlockFileEx 
    WriteFile 
    WriteFileEx 
    WriteFileGather 

The following functions are used with the encrypted file system.

    AddUsersToEncryptedFile 
    CloseEncryptedFileRaw 
    DecryptFile 
    DuplicateEncryptionInfoFile 
    EncryptFile 
    EncryptionDisable 
    FileEncryptionStatus 
    FreeEncryptionCertificateHashList 
    OpenEncryptedFileRaw 
    QueryRecoveryAgentsOnEncryptedFile 
    QueryUsersOnEncryptedFile 
    ReadEncryptedFileRaw 
    RemoveUsersFromEncryptedFile 
    SetUserFileEncryptionKey 
    WriteEncryptedFileRaw 

The following functions are used with the file system redirector.

    Wow64DisableWow64FsRedirection 
    Wow64EnableWow64FsRedirection 
    Wow64RevertWow64FsRedirection 

The following functions are used to decompress files that are compressed by the Lempel-Ziv algorithm.

    GetExpandedName 
    LZClose 
    LZCopy 
    LZInit 
    LZOpenFile 
    LZRead 
    LZSeek 

The following callback functions are used in file I/O.

    CopyProgressRoutine 
    ExportCallback 
    FileIOCompletionRoutine 
    ImportCallback 
"""
blacklisted_words = [line.strip()  for line in  blacklisted_words_txt.split('\n')    if len(line) < 42 and line.strip()]


void_functions = []

suffix = '.cs'

def execute(command, stdin):
  return config.execute(command, stdin, output_encoding='Latin-1')

def compile( jail, dir, slug, code, tag="reference|student", translate_line=None ):
   
    os.makedirs( jail + dir, 0o755, exist_ok=True);
    os.chdir(jail + dir)
    
    # SAVE to FILE
    with open(slug + ".cs", "w") as f:
      f.write(code)
      
    # remove previous compilation result (if such exists)
    if os.path.isfile( slug + '.exe' ):
        os.remove( slug + '.exe' )
    
    
    # COMPILE
    compiler_cmd = [cfg["cs_compiler"],   slug+".cs"]
    compiling = execute(compiler_cmd, "")
    
    def check_compile_err():
        
        if (compiling.stderr != "" or compiling.returncode != 0
        or not os.path.isfile(jail + dir + slug + suffix)):

            if 'student' in tag.lower():   #  humanreadable info for students
                errmsg = re.sub('(\.cs:)(\d+)(?!\d)',   # should  be adapted for C#
                       lambda m : m.group(1)+translate_line(m.group(2)),  
                       compiling.stderr)
                return ("Syntax Error", errmsg)
                
            else:     # for reference compilation - more detailed dump
                return  (
                    "Internal Error (Compiling %s)"%tag ,
                    "cmd:"+pre(" ".join(compiler_cmd))+ # websheet.slug + suffix
                    "<br>stdout:"+pre(compiling.stdout)+
                    "<br>stderr:"+pre(compiling.stderr)+
                    "<br>retval:"+pre(str(compiling.returncode))
                    )                    
                    
    compiling.error  =  check_compile_err()
    return compiling
        
        
def run(slug, tag="reference|student", stdin="" ):
    
    cmd = [cfg["safeexec-executable-abspath"]]
    #~ cmd += ["--chroot_dir", cfg["java_jail-abspath"]]
    #~ cmd += ["--exec_dir", "/" + refdir]
    cmd += ["--fsize", "5"]
    cmd += ["--nproc", "1"]
    cmd += ["--clock", "1"]
    cmd += ["--mem", "600000"]   # TODO: move to config?
    cmd += ["--exec", slug+'.exe']
    #~ cmd += args
    
    running = execute(cmd, stdin)

    def check_run_error():

        if running.returncode != 0 or not running.stderr.startswith("OK"):

            if 'student' in tag.lower():   #  for students -- humanreadable info 
                errmsg = running.stderr
                if "elapsed time:" in errmsg:
                  errmsg = errmsg[:errmsg.index("elapsed time:")]
                errmsg = errmsg.replace("Command terminated by signal (8: SIGFPE)",
                                        "Floating point exception")
                errmsg = errmsg.replace("Command terminated by signal (11: SIGSEGV)",
                                        "Segmentation fault (core dumped)")
                result = "<div>Crashed! "
                if errmsg != "":
                  result += "Error messages:" + pre(errmsg)
                if running.stdout != "":
                  result += "Produced this output:"+pre(running.stdout)
        #        result += "Return code:"+pre(str(running.returncode))
                result += "</div>"
                
                return ("Sandbox Limit", result)

                
            else:   # for reference -- more detailed<commit_msg>applied allowance of utf-8  to C#/Mono (hopefully)<commit_after>import config, json, cgi, sys, Websheet, re, os
from utils import *

cfg = config.config_jo
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa364232(v=vs.85).aspx
blacklisted_words_txt = """
using 
StreamReader
StreamWriter
File
.Read
FileStream
OpenText
System.IO

The following functions are used with file I/O.

    CancelIo 
    CancelIoEx 
    CancelSynchronousIo 
    CreateIoCompletionPort 
    FlushFileBuffers 
    GetQueuedCompletionStatus 
    GetQueuedCompletionStatusEx 
    LockFile 
    LockFileEx 
    PostQueuedCompletionStatus 
    ReadFile 
    ReadFileEx 
    ReadFileScatter 
    SetEndOfFile 
    SetFileCompletionNotificationModes 
    SetFileIoOverlappedRange 
    SetFilePointer 
    SetFilePointerEx 
    UnlockFile 
    UnlockFileEx 
    WriteFile 
    WriteFileEx 
    WriteFileGather 

The following functions are used with the encrypted file system.

    AddUsersToEncryptedFile 
    CloseEncryptedFileRaw 
    DecryptFile 
    DuplicateEncryptionInfoFile 
    EncryptFile 
    EncryptionDisable 
    FileEncryptionStatus 
    FreeEncryptionCertificateHashList 
    OpenEncryptedFileRaw 
    QueryRecoveryAgentsOnEncryptedFile 
    QueryUsersOnEncryptedFile 
    ReadEncryptedFileRaw 
    RemoveUsersFromEncryptedFile 
    SetUserFileEncryptionKey 
    WriteEncryptedFileRaw 

The following functions are used with the file system redirector.

    Wow64DisableWow64FsRedirection 
    Wow64EnableWow64FsRedirection 
    Wow64RevertWow64FsRedirection 

The following functions are used to decompress files that are compressed by the Lempel-Ziv algorithm.

    GetExpandedName 
    LZClose 
    LZCopy 
    LZInit 
    LZOpenFile 
    LZRead 
    LZSeek 

The following callback functions are used in file I/O.

    CopyProgressRoutine 
    ExportCallback 
    FileIOCompletionRoutine 
    ImportCallback 
"""
blacklisted_words = [line.strip()  for line in  blacklisted_words_txt.split('\n')    if len(line) < 42 and line.strip()]


void_functions = []

suffix = '.cs'

from config import execute

def compile( jail, dir, slug, code, tag="reference|student", translate_line=None ):
   
    os.makedirs( jail + dir, 0o755, exist_ok=True);
    os.chdir(jail + dir)
    
    # SAVE to FILE
    with open(slug + ".cs", "w", encoding="utf-8") as f:
      f.write(code)
      
    # remove previous compilation result (if such exists)
    if os.path.isfile( slug + '.exe' ):
        os.remove( slug + '.exe' )
    
    
    # COMPILE
    compiler_cmd = [cfg["cs_compiler"],   slug+".cs"]
    compiling = execute(compiler_cmd, "")
    
    def check_compile_err():
        
        if (compiling.stderr != "" or compiling.returncode != 0
        or not os.path.isfile(jail + dir + slug + suffix)):

            if 'student' in tag.lower():   #  humanreadable info for students
                errmsg = re.sub('(\.cs:)(\d+)(?!\d)',   # should  be adapted for C#
                       lambda m : m.group(1)+translate_line(m.group(2)),  
                       compiling.stderr)
                return ("Syntax Error", errmsg)
                
            else:     # for reference compilation - more detailed dump
                return  (
                    "Internal Error (Compiling %s)"%tag ,
                    "cmd:"+pre(" ".join(compiler_cmd))+ # websheet.slug + suffix
                    "<br>stdout:"+pre(compiling.stdout)+
                    "<br>stderr:"+pre(compiling.stderr)+
                    "<br>retval:"+pre(str(compiling.returncode))
                    )                    
                    
    compiling.error  =  check_compile_err()
    return compiling
        
        
def run(slug, tag="reference|student", stdin="" ):
    
    cmd = [cfg["safeexec-executable-abspath"]]
    #~ cmd += ["--chroot_dir", cfg["java_jail-abspath"]]
    #~ cmd += ["--exec_dir", "/" + refdir]
    cmd += ["--fsize", "5"]
    cmd += ["--nproc", "1"]
    cmd += ["--clock", "1"]
    cmd += ["--mem", "600000"]   # TODO: move to config?
    cmd += ["--exec", slug+'.exe']
    #~ cmd += args
    
    running = execute(cmd, stdin, flag_badchars = True)

    def check_run_error():

        if running.returncode != 0 or not running.stderr.startswith("OK"):

            if 'student' in tag.lower():   #  for students -- humanreadable info 
                errmsg = running.stderr
                if "elapsed time:" in errmsg:
                  errmsg = errmsg[:errmsg.index("elapsed time:")]
                errmsg = errmsg.replace("Command terminated by signal (8: SIGFPE)",
                                        "Floating point exception")
                errmsg = errmsg.replace("Command terminated by signal (11: SIGSEGV)",
                                        "Segmentation fault (core dumped)")
                result = "<div>Crashed! "
                if errmsg != "":
                  result += "Error messages:" + pre(errmsg)
                if running.stdout != "":
                  result += "Produced this output:"+pre(running.stdout)
        #        result += "Return code:"+pre(str(running.returncode))
                result += "</div>"
                
                return ("Sandbox Limit", result)

                
            else:   # for reference -- more detailed
{'fraction_non_alphanumeric': 0.10130287009063445, 'fraction_numerical': 0.008213746223564954, 'mean_word_length': 2.113756613756614, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5467802', 'n_tokens_mistral': 3260, 'n_tokens_neox': 2978, 'n_words': 731}