<commit_before>#!/usr/bin/env python
# encoding: utf-8
# <NAME>, 2010 (ita)

"""
Provide a scanner for finding dependencies on d files
"""

import re
from waflib import Utils, Logs

def filter_comments(filename):
	"""
	:param filename: d file name
	:type filename: string
	:rtype: list
	:return: a list of characters
	"""
	txt = Utils.readf(filename)
	i = 0
	buf = []
	max = len(txt)
	begin = 0
	while i < max:
		c = txt[i]
		if c == '"' or c == "'":  # skip a string or character literal
			buf.append(txt[begin:i])
			delim = c
			i += 1
			while i < max:
				c = txt[i]
				if c == delim: break
				elif c == '\\':  # skip the character following backslash
					i += 1
				i += 1
			i += 1
			begin = i
		elif c == '/':  # try to replace a comment with whitespace
			buf.append(txt[begin:i])
			i += 1
			if i == max: break
			c = txt[i]
			if c == '+':  # eat nesting /+ +/ comment
				i += 1
				nesting = 1
				c = None
				while i < max:
					prev = c
					c = txt[i]
					if prev == '/' and c == '+':
						nesting += 1
						c = None
					elif prev == '+' and c == '/':
						nesting -= 1
						if nesting == 0: break
						c = None
					i += 1
			elif c == '*':  # eat /* */ comment
				i += 1
				c = None
				while i < max:
					prev = c
					c = txt[i]
					if prev == '*' and c == '/': break
					i += 1
			elif c == '/':  # eat // comment
				i += 1
				while i < max and txt[i] != '\n':
					i += 1
			else:  # no comment
				begin = i - 1
				continue
			i += 1
			begin = i
			buf.append(' ')
		else:
			i += 1
	buf.append(txt[begin:])
	return buf

class d_parser(object):
	"""
	Parser for d files
	"""
	def __init__(self, env, incpaths):
		#self.code = ''
		#self.module = ''
		#self.imports = []

		self.allnames = []

		self.re_module = re.compile("module\s+([^;]+)")
		self.re_import = re.compile("import\s+([^;]+)")
		self.re_import_bindings = re.compile("([^:]+):(.*)")
		self.re_import_alias = re.compile("[^=]+=(.+)")

		self.env = env

		self.nodes = []
		self.names = []

		self.incpaths = incpaths

	def tryfind(self, filename):
		"""
		Search file a file matching an module/import directive

		:param filename: file to read
		:type filename: string
		"""
		file_path = filename.replace('.', '/')
		for n in self.incpaths:
			for suffix in ('.d', '/package.d'):
				found = n.find_resource(file_path + suffix)
				if found:
					self.nodes.append(found)
					self.waiting.append(found)
					return

		if not filename in self.names:
			self.names.append(filename)

	def get_strings(self, code):
		"""
		:param code: d code to parse
		:type code: string
		:return: the modules that the code uses
		:rtype: a list of match objects
		"""
		#self.imports = []<commit_msg>Revert "Make D dependency scanner aware of package.d files"<commit_after>#!/usr/bin/env python
# encoding: utf-8
# <NAME>, 2010 (ita)

"""
Provide a scanner for finding dependencies on d files
"""

import re
from waflib import Utils, Logs

def filter_comments(filename):
	"""
	:param filename: d file name
	:type filename: string
	:rtype: list
	:return: a list of characters
	"""
	txt = Utils.readf(filename)
	i = 0
	buf = []
	max = len(txt)
	begin = 0
	while i < max:
		c = txt[i]
		if c == '"' or c == "'":  # skip a string or character literal
			buf.append(txt[begin:i])
			delim = c
			i += 1
			while i < max:
				c = txt[i]
				if c == delim: break
				elif c == '\\':  # skip the character following backslash
					i += 1
				i += 1
			i += 1
			begin = i
		elif c == '/':  # try to replace a comment with whitespace
			buf.append(txt[begin:i])
			i += 1
			if i == max: break
			c = txt[i]
			if c == '+':  # eat nesting /+ +/ comment
				i += 1
				nesting = 1
				c = None
				while i < max:
					prev = c
					c = txt[i]
					if prev == '/' and c == '+':
						nesting += 1
						c = None
					elif prev == '+' and c == '/':
						nesting -= 1
						if nesting == 0: break
						c = None
					i += 1
			elif c == '*':  # eat /* */ comment
				i += 1
				c = None
				while i < max:
					prev = c
					c = txt[i]
					if prev == '*' and c == '/': break
					i += 1
			elif c == '/':  # eat // comment
				i += 1
				while i < max and txt[i] != '\n':
					i += 1
			else:  # no comment
				begin = i - 1
				continue
			i += 1
			begin = i
			buf.append(' ')
		else:
			i += 1
	buf.append(txt[begin:])
	return buf

class d_parser(object):
	"""
	Parser for d files
	"""
	def __init__(self, env, incpaths):
		#self.code = ''
		#self.module = ''
		#self.imports = []

		self.allnames = []

		self.re_module = re.compile("module\s+([^;]+)")
		self.re_import = re.compile("import\s+([^;]+)")
		self.re_import_bindings = re.compile("([^:]+):(.*)")
		self.re_import_alias = re.compile("[^=]+=(.+)")

		self.env = env

		self.nodes = []
		self.names = []

		self.incpaths = incpaths

	def tryfind(self, filename):
		"""
		Search file a file matching an module/import directive

		:param filename: file to read
		:type filename: string
		"""
		found = 0
		for n in self.incpaths:
			found = n.find_resource(filename.replace('.', '/') + '.d')
			if found:
				self.nodes.append(found)
				self.waiting.append(found)
				break
		if not found:
			if not filename in self.names:
				self.names.append(filename)

	def get_strings(self, code):
		"""
		:param code: d code to parse
		:type code: string
		:return: the modules that the code uses
		:rtype: a list of match objects
		"""
		#self.imports = []
{'fraction_non_alphanumeric': 0.1653689284390063, 'fraction_numerical': 0.009454949944382647, 'mean_word_length': 2.3037354562155543, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5488072', 'n_tokens_mistral': 2455, 'n_tokens_neox': 1949, 'n_words': 612}