<commit_before>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2011 <NAME> for LOCO (LOOHUIS CONSULTING)
## Licensed under Apache 2.0, see LICENSE file for details

'''
This tool stores strings extracted from source code into a knowledgebase.
'''

import sys, os, string, re, subprocess, magic
from optparse import OptionParser
import sqlite3

ms = magic.open(magic.MAGIC_NONE)
ms.load()

def extractsourcestrings(srcdir, sqldb, package, pversion):
        srcdirlen = len(srcdir)+1
        osgen = os.walk(srcdir)

        try:
                while True:
                        i = osgen.next()
                        for p in i[2]:
				## we're only interested in a few files right now, will add more in the future
				## some filenames might have uppercase extensions, so lowercase them first
				p_nocase = p.lower()
				if p_nocase.endswith('.c') or p_nocase.endswith('.h') or p_nocase.endswith('.cpp'):
					## Remove all C and C++ style comments. If a file is in iso-8859-1
					## instead of ASCII or UTF-8 we need to do some extra work by
					## converting it first using iconv.
					## This is not failsafe, because magic gets it wrong sometimes, so we
					## need some way to kill the subprocess if it is running too long.
					#print p
					datatype = ms.file("%s/%s" % ( i[0], p))
					print >>sys.stderr, "%s/%s" % (i[0], p), " ", datatype
					if "AppleDouble" in datatype:
						continue
					if "ISO-8859" in datatype:
						src = open("%s/%s" % (i[0], p)).read()
						p1 = subprocess.Popen(["iconv", "-f", "latin1", "-t", "utf-8"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
						cleanedup_src = p1.communicate(src)[0]
						p2 = subprocess.Popen(['./remccoms3.sed'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,close_fds=True)
						(stanout, stanerr) = p2.communicate(cleanedup_src)<commit_msg>scan more files<commit_after>#!/usr/bin/python

## Binary Analysis Tool
## Copyright 2009-2011 <NAME> for LOCO (LOOHUIS CONSULTING)
## Licensed under Apache 2.0, see LICENSE file for details

'''
This tool stores strings extracted from source code into a knowledgebase.
'''

import sys, os, string, re, subprocess, magic
from optparse import OptionParser
import sqlite3

ms = magic.open(magic.MAGIC_NONE)
ms.load()

def extractsourcestrings(srcdir, sqldb, package, pversion):
        srcdirlen = len(srcdir)+1
        osgen = os.walk(srcdir)

        try:
                while True:
                        i = osgen.next()
                        for p in i[2]:
				## we're only interested in a few files right now, will add more in the future
				## some filenames might have uppercase extensions, so lowercase them first
				p_nocase = p.lower()
				if p_nocase.endswith('.c') or p_nocase.endswith('.h') or p_nocase.endswith('.cpp') or p_nocase.endswith('.cc') or p_nocase.endswith('.hh'):
					## Remove all C and C++ style comments. If a file is in iso-8859-1
					## instead of ASCII or UTF-8 we need to do some extra work by
					## converting it first using iconv.
					## This is not failsafe, because magic gets it wrong sometimes, so we
					## need some way to kill the subprocess if it is running too long.
					#print p
					datatype = ms.file("%s/%s" % ( i[0], p))
					print >>sys.stderr, "%s/%s" % (i[0], p), " ", datatype
					if "AppleDouble" in datatype:
						continue
					if "ISO-8859" in datatype:
						src = open("%s/%s" % (i[0], p)).read()
						p1 = subprocess.Popen(["iconv", "-f", "latin1", "-t", "utf-8"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
						cleanedup_src = p1.communicate(src)[0]
						p2 = subprocess.Popen(['./remccoms3.sed'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,close_fds=True)
						(stanout, stanerr) = p2.communicate(cleanedup_src)
{'fraction_non_alphanumeric': 0.12836767036450078, 'fraction_numerical': 0.017960908610670893, 'mean_word_length': 3.0074074074074075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369487', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1249, 'n_words': 399}