<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2010-2012 Appcelerator, Inc. All Rights Reserved.
# Licensed under the Apache Public License (version 2)
#
# parse out Titanium API documentation templates into a 
# format that can be used by other documentation generators
# such as PDF, etc.
import os, sys, traceback
import re, optparse
import generators
from distutils.version import StrictVersion
from common import lazyproperty, dict_has_non_empty_member, not_real_titanium_types, DEFAULT_PLATFORMS, pretty_platform_name, first_version_for_platform

try:
	import yaml
except:
	print >> sys.stderr, "You don't have pyyaml!\n"
	print >> sys.stderr, "You can install it with:\n"
	print >> sys.stderr, ">  sudo easy_install pyyaml\n"
	print >> sys.stderr, ""
	sys.exit(1)

this_dir = os.path.dirname(os.path.abspath(__file__))

# We package markdown and mako in support/common.
common_support_dir = os.path.abspath(os.path.join(this_dir, "..", "support", "common"))
sys.path.append(common_support_dir)
import markdown
from mako.template import Template

# TiLogger is in support/android
android_support_dir = os.path.abspath(os.path.join(this_dir, "..", "support", "android"))
sys.path.append(android_support_dir)
from tilogger import *
log = TiLogger(None)

DEFAULT_SINCE = "0.8"
apis = {} # raw conversion from yaml
annotated_apis = {} # made friendlier for templates, etc.
current_api = None
ignore_dirs = (".git", ".svn", "CVS", "node_modules")
ignore_files = ("template.yml")
warn_inherited = False # see optparse option with same name in main()
options = None

def has_ancestor(one_type, ancestor_name):
	if one_type["name"] == ancestor_name:
		return True
	if "extends" in one_type and one_type["extends"] == ancestor_name:
		return True
	elif "extends" not in one_type:
		return False
	else:
		parent_type_name = one_type["extends"]
		if (parent_type_name is None or not isinstance(parent_type_name, basestring) or
				parent_type_name.lower() == "object"):
			return False
		if not parent_type_name in apis:
			log.warn("%s extends %s but %s type information not found" % (one_type["name"],
				parent_type_name, parent_type_name))
			return False
		return has_ancestor(apis[parent_type_name], ancestor_name)

def is_titanium_module(one_type):
	return has_ancestor(one_type, "Titanium.Module")

def is_titanium_proxy(one_type):
	# When you use this, don't forget that modules are also proxies
	return has_ancestor(one_type, "Titanium.Proxy")

# TIDOC-1080: Support exclude-platforms tag
def exclude_platforms(obj, parent):
	platforms = None
	if parent is not None:
		all_platforms = None
		if dict_has_non_empty_member(parent.api_obj, "platforms"):
			all_platforms = parent.api_obj["platforms"]
		else:
			all_platforms = DEFAULT_PLATFORMS
		if all_platforms is not None:
			platforms = [x for x in all_platforms if x not in obj["exclude-platforms"]]
	if not platforms:
		log.warn("No platforms for %s" % (obj["name"]))
	return platforms

def combine_platforms_and_since(annotated_obj):
	parent = annotated_obj.parent
	obj = annotated_obj.api_obj
	result = []
	platforms = None
	since = DEFAULT_SINCE
	if dict_has_non_empty_member(obj, "platforms"):
		platforms = obj["platforms"]
	# Method/property/event can't have more platforms than the types they belong to.
	if (platforms is None or
			isinstance(annotated_obj, AnnotatedMethod) or isinstance(annotated_obj, AnnotatedProperty) or
			isinstance(annotated_obj, AnnotatedEvent)):
		if parent is not None:
			if dict_has_non_empty_member(parent.api_obj, "platforms"):
				if platforms is None or len(parent.api_obj["platforms"]) < len(platforms):
					platforms = parent.api_obj["platforms"]
	# Last resort is the default list of platforms
	if platforms is None:
		platforms = DEFAULT_PLATFORMS
	if "since" in obj and len(obj["since"]) > 0:
		since = obj["since"]
	else:
		# If a method/event/property we can check type's "since"
		if (isinstance(annotated_obj, AnnotatedMethod) or isinstance(annotated_obj, AnnotatedProperty) or
				isinstance(annotated_obj, AnnotatedEvent)):
			if (parent is not None and
					dict_has_non_empty_member(parent.api_obj, "since")):
				since = parent.api_obj["since"]

	since_is_dict = isinstance(since, dict)
	for name in platforms:
		one_platform = {"name": name, "pretty_name": pretty_platform_name(name)}
		min_since = first_version_for_platform(one_platform["name"])
		if not since_is_dict:
			one_platform["since"] = since
			if min_since is not None:
				if StrictVersion(since) < StrictVersion(min_since):
					one_platform["since"] = min_since
		else:
			if name in since:
				one_platform["since"] = since[name]
			else:
				if min_since is not None:
					one_platform["since"] = min_since
				else:
					one_platform["since"] = DEFAULT_SINCE
		result.append(one_platform)

	# Be sure no "since" is _before_ a parent object since.
	if parent and parent.platforms:
		for entry in result:
			platform_name = entry["name"]
			version_parts = entry["since"].split(".")
			for parent_entry in parent.platforms:
				if parent_entry["name"] == platform_name:
					parent_version_parts = parent_entry["since"].split(".")
					if parent_version_parts > version_parts:
						entry["since"] = parent_entry["since"]
					break
	return result

def load_one_yaml(filepath):
	f = None
	try:
		f = open(filepath, "r")
		types = [the_type for the_type in yaml.load_all(f)]
		return types
	except KeyboardInterrupt:
		raise
	except:
		log.error("Exception occurred while processing %s:" % filepath)
		raise
	finally:
		if f is not None:
			try:
				f.close()
			except:
				pass

def load_generators(options):
	for output_type in options.formats.split(","):
		try:
			__import__("generators.%s_generator" % output_type)
		except:
			log.error("Output format %s is not recognized" % output_type)
			sys.exit(1)

def generate_output(options):
	for output_type in options.formats.split(","):
		if annotated_apis is None or len(annotated_apis) == 0:
			annotate_apis()
		generator = getattr(generators, "%s_generator" % output_type)
		generator.generate(apis, annotated_apis, options)

def process_yaml(source_dirs, options=None):
	global apis
	log.info("Parsing YAML files")
	
	if options and options.exclude_external and len(source_dirs) <= 1:
		log.error("At least one path must be passed when using -e or --exclude-external")
		sys.exit(1)
	tag_external = True if options and options.exclude_external else False
	
	for source_dir in source_dirs:
		for root, dirs, files in os.walk(source_dir):
			for name in ignore_dirs:
				if name in dirs:
					dirs.remove(name) # don't visit ignored directoriess
			for filename in files:
				if os.path.splitext(filename)[-1] != ".yml" or filename in ignore_files:
					continue
				filepath = os.path.join(root, filename)
				relative_path = os.path.relpath(filepath)
				log.trace("Processing: %s" % filepath)
				types = None
				types = load_one_yaml(filepath)
				if types is None:
					log.trace("%s skipped" % filepath)
				else:
					for one_type in types:
						if one_type["name"] in apis:
							log.warn("%s has a duplicate" % one_type["name"])
						one_type["external"] = tag_external
						one_type["filepath"] = relative_path
						apis[one_type["name"]] = one_type
		tag_external = False

# If documentation for a method/event/property only "partially overrides"
# the documentation for the super type, this will fill in the rest of
# the documentation by inheriting it from the super type.
def finish_partial_overrides():
	global apis
	log.trace("Finishing partial overrides")<commit_msg>Now save absolute path to file with each class member, instead of relative path.<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2010-2012 Appcelerator, Inc. All Rights Reserved.
# Licensed under the Apache Public License (version 2)
#
# parse out Titanium API documentation templates into a 
# format that can be used by other documentation generators
# such as PDF, etc.
import os, sys, traceback
import re, optparse
import generators
from distutils.version import StrictVersion
from common import lazyproperty, dict_has_non_empty_member, not_real_titanium_types, DEFAULT_PLATFORMS, pretty_platform_name, first_version_for_platform

try:
	import yaml
except:
	print >> sys.stderr, "You don't have pyyaml!\n"
	print >> sys.stderr, "You can install it with:\n"
	print >> sys.stderr, ">  sudo easy_install pyyaml\n"
	print >> sys.stderr, ""
	sys.exit(1)

this_dir = os.path.dirname(os.path.abspath(__file__))

# We package markdown and mako in support/common.
common_support_dir = os.path.abspath(os.path.join(this_dir, "..", "support", "common"))
sys.path.append(common_support_dir)
import markdown
from mako.template import Template

# TiLogger is in support/android
android_support_dir = os.path.abspath(os.path.join(this_dir, "..", "support", "android"))
sys.path.append(android_support_dir)
from tilogger import *
log = TiLogger(None)

DEFAULT_SINCE = "0.8"
apis = {} # raw conversion from yaml
annotated_apis = {} # made friendlier for templates, etc.
current_api = None
ignore_dirs = (".git", ".svn", "CVS", "node_modules")
ignore_files = ("template.yml")
warn_inherited = False # see optparse option with same name in main()
options = None

def has_ancestor(one_type, ancestor_name):
	if one_type["name"] == ancestor_name:
		return True
	if "extends" in one_type and one_type["extends"] == ancestor_name:
		return True
	elif "extends" not in one_type:
		return False
	else:
		parent_type_name = one_type["extends"]
		if (parent_type_name is None or not isinstance(parent_type_name, basestring) or
				parent_type_name.lower() == "object"):
			return False
		if not parent_type_name in apis:
			log.warn("%s extends %s but %s type information not found" % (one_type["name"],
				parent_type_name, parent_type_name))
			return False
		return has_ancestor(apis[parent_type_name], ancestor_name)

def is_titanium_module(one_type):
	return has_ancestor(one_type, "Titanium.Module")

def is_titanium_proxy(one_type):
	# When you use this, don't forget that modules are also proxies
	return has_ancestor(one_type, "Titanium.Proxy")

# TIDOC-1080: Support exclude-platforms tag
def exclude_platforms(obj, parent):
	platforms = None
	if parent is not None:
		all_platforms = None
		if dict_has_non_empty_member(parent.api_obj, "platforms"):
			all_platforms = parent.api_obj["platforms"]
		else:
			all_platforms = DEFAULT_PLATFORMS
		if all_platforms is not None:
			platforms = [x for x in all_platforms if x not in obj["exclude-platforms"]]
	if not platforms:
		log.warn("No platforms for %s" % (obj["name"]))
	return platforms

def combine_platforms_and_since(annotated_obj):
	parent = annotated_obj.parent
	obj = annotated_obj.api_obj
	result = []
	platforms = None
	since = DEFAULT_SINCE
	if dict_has_non_empty_member(obj, "platforms"):
		platforms = obj["platforms"]
	# Method/property/event can't have more platforms than the types they belong to.
	if (platforms is None or
			isinstance(annotated_obj, AnnotatedMethod) or isinstance(annotated_obj, AnnotatedProperty) or
			isinstance(annotated_obj, AnnotatedEvent)):
		if parent is not None:
			if dict_has_non_empty_member(parent.api_obj, "platforms"):
				if platforms is None or len(parent.api_obj["platforms"]) < len(platforms):
					platforms = parent.api_obj["platforms"]
	# Last resort is the default list of platforms
	if platforms is None:
		platforms = DEFAULT_PLATFORMS
	if "since" in obj and len(obj["since"]) > 0:
		since = obj["since"]
	else:
		# If a method/event/property we can check type's "since"
		if (isinstance(annotated_obj, AnnotatedMethod) or isinstance(annotated_obj, AnnotatedProperty) or
				isinstance(annotated_obj, AnnotatedEvent)):
			if (parent is not None and
					dict_has_non_empty_member(parent.api_obj, "since")):
				since = parent.api_obj["since"]

	since_is_dict = isinstance(since, dict)
	for name in platforms:
		one_platform = {"name": name, "pretty_name": pretty_platform_name(name)}
		min_since = first_version_for_platform(one_platform["name"])
		if not since_is_dict:
			one_platform["since"] = since
			if min_since is not None:
				if StrictVersion(since) < StrictVersion(min_since):
					one_platform["since"] = min_since
		else:
			if name in since:
				one_platform["since"] = since[name]
			else:
				if min_since is not None:
					one_platform["since"] = min_since
				else:
					one_platform["since"] = DEFAULT_SINCE
		result.append(one_platform)

	# Be sure no "since" is _before_ a parent object since.
	if parent and parent.platforms:
		for entry in result:
			platform_name = entry["name"]
			version_parts = entry["since"].split(".")
			for parent_entry in parent.platforms:
				if parent_entry["name"] == platform_name:
					parent_version_parts = parent_entry["since"].split(".")
					if parent_version_parts > version_parts:
						entry["since"] = parent_entry["since"]
					break
	return result

def load_one_yaml(filepath):
	f = None
	try:
		f = open(filepath, "r")
		types = [the_type for the_type in yaml.load_all(f)]
		return types
	except KeyboardInterrupt:
		raise
	except:
		log.error("Exception occurred while processing %s:" % filepath)
		raise
	finally:
		if f is not None:
			try:
				f.close()
			except:
				pass

def load_generators(options):
	for output_type in options.formats.split(","):
		try:
			__import__("generators.%s_generator" % output_type)
		except:
			log.error("Output format %s is not recognized" % output_type)
			sys.exit(1)

def generate_output(options):
	for output_type in options.formats.split(","):
		if annotated_apis is None or len(annotated_apis) == 0:
			annotate_apis()
		generator = getattr(generators, "%s_generator" % output_type)
		generator.generate(apis, annotated_apis, options)

def process_yaml(source_dirs, options=None):
	global apis
	log.info("Parsing YAML files")
	
	if options and options.exclude_external and len(source_dirs) <= 1:
		log.error("At least one path must be passed when using -e or --exclude-external")
		sys.exit(1)
	tag_external = True if options and options.exclude_external else False
	
	for source_dir in source_dirs:
		for root, dirs, files in os.walk(source_dir):
			for name in ignore_dirs:
				if name in dirs:
					dirs.remove(name) # don't visit ignored directoriess
			for filename in files:
				if os.path.splitext(filename)[-1] != ".yml" or filename in ignore_files:
					continue
				filepath = os.path.join(root, filename)
				log.trace("Processing: %s" % filepath)
				types = None
				types = load_one_yaml(filepath)
				if types is None:
					log.trace("%s skipped" % filepath)
				else:
					for one_type in types:
						if one_type["name"] in apis:
							log.warn("%s has a duplicate" % one_type["name"])
						one_type["external"] = tag_external
						one_type["filepath"] = filepath
						apis[one_type["name"]] = one_type
		tag_external = False

# If documentation for a method/event/property only "partially overrides"
# the documentation for the super type, this will fill in the rest of
# the documentation by inheriting it from the super type.
def finish_partial_overrides():
	global apis
	log.trace("Finishing partial overrides")
{'fraction_non_alphanumeric': 0.1043322734499205, 'fraction_numerical': 0.0029146793852676206, 'mean_word_length': 4.654307116104869, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2174141', 'n_tokens_mistral': 5483, 'n_tokens_neox': 4915, 'n_words': 1424}