<commit_before>#
#   Pyrex Scanner
#

#import pickle
import cPickle as pickle

import os
import platform
import stat
import sys
from time import time

import cython
cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object)

from Cython import Plex, Utils
from Cython.Plex.Scanners import Scanner
from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error
from Lexicon import string_prefixes, raw_prefixes, make_lexicon, IDENT

from StringEncoding import EncodedString

try:
    plex_version = Plex._version
except AttributeError:
    plex_version = None
#print "Plex version:", plex_version ###

debug_scanner = 0
trace_scanner = 0
scanner_debug_flags = 0
scanner_dump_file = None
binary_lexicon_pickle = 1
notify_lexicon_unpickling = 0
notify_lexicon_pickling = 1

lexicon = None

#-----------------------------------------------------------------

def hash_source_file(path):
    # Try to calculate a hash code for the given source file.
    # Returns an empty string if the file cannot be accessed.
    #print "Hashing", path ###
    try:
        from hashlib import md5 as new_md5
    except ImportError:
        from md5 import new as new_md5
    try:
        try:
            f = open(path, "rU")
            text = f.read()
        except IOError, e:
            print("Unable to hash scanner source file (%s)" % e)
            return ""
    finally:
        f.close()
    # Normalise spaces/tabs. We don't know what sort of
    # space-tab substitution the file may have been
    # through, so we replace all spans of spaces and
    # tabs by a single space.
    import re
    text = re.sub("[ \t]+", " ", text)
    hash = new_md5(text).hexdigest()
    return hash

def open_pickled_lexicon(expected_hash):
    # Try to open pickled lexicon file and verify that
    # it matches the source file. Returns the opened
    # file if successful, otherwise None. ???
    global lexicon_pickle
    f = None
    result = None
    if os.path.exists(lexicon_pickle):
        try:
            f = open(lexicon_pickle, "rb")
            actual_hash = pickle.load(f)
            if actual_hash == expected_hash:
                result = f
                f = None
            else:
                print("Lexicon hash mismatch:")       ###
                print("   expected " + expected_hash) ###
                print("   got     " + actual_hash)    ###
        except IOError, e:
            print("Warning: Unable to read pickled lexicon " + lexicon_pickle)
            print(e)
    if f:
        f.close()
    return result
<commit_msg>More unpickling error catching.<commit_after>#
#   Pyrex Scanner
#

#import pickle
import cPickle as pickle

import os
import platform
import stat
import sys
from time import time

import cython
cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object)

from Cython import Plex, Utils
from Cython.Plex.Scanners import Scanner
from Cython.Plex.Errors import UnrecognizedInput
from Errors import CompileError, error
from Lexicon import string_prefixes, raw_prefixes, make_lexicon, IDENT

from StringEncoding import EncodedString

try:
    plex_version = Plex._version
except AttributeError:
    plex_version = None
#print "Plex version:", plex_version ###

debug_scanner = 0
trace_scanner = 0
scanner_debug_flags = 0
scanner_dump_file = None
binary_lexicon_pickle = 1
notify_lexicon_unpickling = 0
notify_lexicon_pickling = 1

lexicon = None

#-----------------------------------------------------------------

def hash_source_file(path):
    # Try to calculate a hash code for the given source file.
    # Returns an empty string if the file cannot be accessed.
    #print "Hashing", path ###
    try:
        from hashlib import md5 as new_md5
    except ImportError:
        from md5 import new as new_md5
    try:
        try:
            f = open(path, "rU")
            text = f.read()
        except IOError, e:
            print("Unable to hash scanner source file (%s)" % e)
            return ""
    finally:
        f.close()
    # Normalise spaces/tabs. We don't know what sort of
    # space-tab substitution the file may have been
    # through, so we replace all spans of spaces and
    # tabs by a single space.
    import re
    text = re.sub("[ \t]+", " ", text)
    hash = new_md5(text).hexdigest()
    return hash

def open_pickled_lexicon(expected_hash):
    # Try to open pickled lexicon file and verify that
    # it matches the source file. Returns the opened
    # file if successful, otherwise None. ???
    global lexicon_pickle
    f = None
    result = None
    if os.path.exists(lexicon_pickle):
        try:
            f = open(lexicon_pickle, "rb")
            actual_hash = pickle.load(f)
            if actual_hash == expected_hash:
                result = f
                f = None
            else:
                print("Lexicon hash mismatch:")       ###
                print("   expected " + expected_hash) ###
                print("   got     " + actual_hash)    ###
        except (IOError, pickle.UnpicklingError), e:
            print("Warning: Unable to read pickled lexicon " + lexicon_pickle)
            print(e)
    if f:
        f.close()
    return result

{'fraction_non_alphanumeric': 0.10038095238095238, 'fraction_numerical': 0.004190476190476191, 'mean_word_length': 2.308758664146188, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7527918', 'n_tokens_mistral': 1575, 'n_tokens_neox': 1520, 'n_words': 516}