<commit_before># $Id: XRefCompiler.py,v 1.2 2002/10/29 12:43:25 chalky Exp $
#
# This file is a part of Synopsis.
# Copyright (C) 2000, 2001 <NAME>
# Copyright (C) 2000, 2001 <NAME>
#
# Synopsis is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Log: XRefCompiler.py,v $
# Revision 1.2  2002/10/29 12:43:25  chalky
# Added no_locals support which is on by default, and strips local variables
# from cross reference tables
#
# Revision 1.1  2002/10/29 07:34:29  chalky
# New linker module, compiles xref files into data structure
#
#

import string, cPickle, urllib

from Synopsis.Core import AST, Type, Util

from Linker import config, Operation

def do_compile(input_files, output_file, no_locals):
    # Init data structures
    data = {}
    index = {}
    file_data = (data, index)

    # Read input files
    for file in input_files:
	if config.verbose: print "XRefCompiler: Reading",file
	f = open(file, 'rt')
	lines = f.readlines()
	f.close()

	for line in lines:
	    target, file, line, scope, context = string.split(line)
	    target = tuple(map(intern, string.split(urllib.unquote(target), '\t')))
	    scope = map(intern, string.split(urllib.unquote(scope), '\t'))
	    line = int(line)
	    file = intern(file)

	    if no_locals:
		bad = 0
		for i in range(len(target)):
		    if len(target[i]) > 0 and target[i][0] == '`':
			# Don't store local function variables
			bad = 1
			break
		if bad: continue

	    for i in range(len(scope)):
		if len(scope[i]) > 0 and scope[i][0] == '`':
		    # Function scope, truncate here
		    del scope[i+1:]<commit_msg>Tolerate file not found errors<commit_after># $Id: XRefCompiler.py,v 1.3 2002/11/03 06:08:56 chalky Exp $
#
# This file is a part of Synopsis.
# Copyright (C) 2000, 2001 <NAME>
# Copyright (C) 2000, 2001 <NAME>
#
# Synopsis is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
# 02111-1307, USA.
#
# $Log: XRefCompiler.py,v $
# Revision 1.3  2002/11/03 06:08:56  chalky
# Tolerate file not found errors
#
# Revision 1.2  2002/10/29 12:43:25  chalky
# Added no_locals support which is on by default, and strips local variables
# from cross reference tables
#
# Revision 1.1  2002/10/29 07:34:29  chalky
# New linker module, compiles xref files into data structure
#
#

import string, cPickle, urllib

from Synopsis.Core import AST, Type, Util

from Linker import config, Operation

def do_compile(input_files, output_file, no_locals):
    # Init data structures
    data = {}
    index = {}
    file_data = (data, index)

    # Read input files
    for file in input_files:
	if config.verbose: print "XRefCompiler: Reading",file
	try:
	    f = open(file, 'rt')
	except IOError, e:
	    print "Error opening %s: %s"%(file, e)
	    continue

	lines = f.readlines()
	f.close()

	for line in lines:
	    target, file, line, scope, context = string.split(line)
	    target = tuple(map(intern, string.split(urllib.unquote(target), '\t')))
	    scope = map(intern, string.split(urllib.unquote(scope), '\t'))
	    line = int(line)
	    file = intern(file)

	    if no_locals:
		bad = 0
		for i in range(len(target)):
		    if len(target[i]) > 0 and target[i][0] == '`':
			# Don't store local function variables
			bad = 1
			break
		if bad: continue

	    for i in range(len(scope)):
		if len(scope[i]) > 0 and scope[i][0] == '`':
		    # Function scope, truncate here
		    del scope[i+1:]
{'fraction_non_alphanumeric': 0.10540013012361743, 'fraction_numerical': 0.04077206679679028, 'mean_word_length': 3.579940417080437, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4421688', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1460, 'n_words': 584}