<commit_before>package com.liferay.alloy.tools.tagbuilder;

import com.liferay.alloy.tools.model.Attribute;
import com.liferay.alloy.tools.model.Component;
import com.liferay.portal.freemarker.FreeMarkerUtil;
import com.liferay.portal.kernel.util.FileUtil;
import com.liferay.portal.kernel.util.GetterUtil;
import com.liferay.portal.kernel.util.StringPool;
import com.liferay.portal.kernel.util.StringUtil;
import com.liferay.portal.kernel.xml.Document;
import com.liferay.portal.kernel.xml.Element;
import com.liferay.portal.kernel.xml.SAXReaderUtil;
import com.liferay.portal.util.FileImpl;
import com.liferay.portal.xml.SAXReaderImpl;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * <a href="TagBuilder.java.html"><b><i>View Source</i></b></a>
 *
 * @author <NAME>
 * @author <NAME>
 */
public class TagBuilder {

	public TagBuilder(
			String componentsXML, String componentsExtXML, String templatesDir,
			String javaOutputDir, String javaOutputPackage,	String jspDir,
			String jspOutputDir, String jspCommonInitPath,
			String jspCommonInitOutputDir, String tldDir)
		throws Exception {

		if (SAXReaderUtil.getSAXReader() == null) {
			(new SAXReaderUtil()).setSAXReader(new SAXReaderImpl());
		}

		_componentsExtXML = Arrays.asList(StringUtil.split(componentsExtXML));
		_componentsXML = componentsXML;
		_templatesDir = templatesDir;
		_javaOutputDir = javaOutputDir;
		_javaOutputPackage = javaOutputPackage;
		_jspDir = jspDir;
		_jspOutputDir = jspOutputDir;
		_jspCommonInitPath = jspCommonInitPath;
		_jspCommonInitOutputDir = jspCommonInitOutputDir;
		_tldDir = tldDir;

		_tplTld = _templatesDir + "tld.ftl";
		_tplTag = _templatesDir + "tag.ftl";
		_tplTagBase = _templatesDir + "tag_base.ftl";
		_tplCommonInitJsp = _templatesDir + "common_init_jsp.ftl";
		_tplJsp = _templatesDir + "jsp.ftl";
		_tplInitJsp = _templatesDir + "init_jsp.ftl";
		_tplStartJsp = _templatesDir + "start_jsp.ftl";

		_componentsExtDoc = new ArrayList<Document>();

		_componentsDoc = SAXReaderUtil.read(new File(_componentsXML));

		for (String componentExtXML : _componentsExtXML) {
			File extFile = new File(componentExtXML);

			if (extFile.exists()) {
				_componentsExtDoc.add(SAXReaderUtil.read(extFile));
			}
		}

		_create();
	}

	public static void main(String[] args) throws Exception {
		String componentsXML = System.getProperty("tagbuilder.components.xml");
		String componentsExtXML = System.getProperty("tagbuilder.components.ext.xml");
		String templatesDir = System.getProperty("tagbuilder.templates.dir");
		String javaOutputDir = System.getProperty("tagbuilder.java.output.dir");
		String javaOutputPackage = System.getProperty("tagbuilder.java.output.package");
		String jspDir = System.getProperty("tagbuilder.jsp.dir");
		String jspCommonInitOutputDir = System.getProperty("tagbuilder.jsp.common.init.output.dir");
		String jspCommonInitPath = System.getProperty("tagbuilder.jsp.common.init.path");
		String jspOutputDir = System.getProperty("tagbuilder.jsp.output.dir");
		String tldDir = System.getProperty("tagbuilder.tld.dir");

		new TagBuilder(
			componentsXML, componentsExtXML, templatesDir, javaOutputDir,
			javaOutputPackage, jspDir, jspOutputDir, jspCommonInitPath,
			jspCommonInitOutputDir,  tldDir);
	}

	public Map<String, Object> getDefaultTemplateContext() {
		Map<String, Object> context = new HashMap<String, Object>();

		context.put("authors", AUTHORS);
		context.put("jspCommonInitPath", _jspCommonInitPath);
		context.put("jspDir", _jspDir);
		context.put("packagePath", _javaOutputPackage);

		return context;
	}

	public String getJavaOutputBaseDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputDir(component));
		sb.append(_BASE);
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJavaOutputDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_javaOutputDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJspDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_jspDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJspOutputDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_jspOutputDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public Map<String, Object> getTemplateContext(Component component) {
		Map<String, Object> context = getDefaultTemplateContext();

		String jspRelativePath = getJspDir(component).concat(
			component.getUncamelizedName(StringPool.UNDERLINE));

		context.put("component", component);
		context.put("namespace", component.getAttributeNamespace());
		context.put("jspRelativePath", jspRelativePath);

		return context;
	}

	private void _create() throws Exception {
		List<Component> components = _getAllComponents();

		for (Component component : components) {
			Map<String, Object> context = getTemplateContext(component);

			_createBaseTag(component, context);
			_createPageJSP(component, context);
			_createTag(component, context);
		}

		_createCommonInitJSP();
		_createTld();
	}

	private void _createBaseTag(
			Component component, Map<String, Object> context)
		throws Exception {

		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputBaseDir(component));
		sb.append(_BASE_CLASS_PREFIX);
		sb.append(component.getSafeName());
		sb.append(_CLASS_SUFFIX);

		String content = _processTemplate(_tplTagBase, context);

		File tagFile = new File(sb.toString());

		_writeFile(tagFile, content);
	}

	private void _createCommonInitJSP() throws Exception {
		Map<String, Object> context = getDefaultTemplateContext();

		String contentCommonInitJsp = _processTemplate(_tplCommonInitJsp, context);

		File commonInitFile = new File(_jspCommonInitOutputDir);
		_writeFile(commonInitFile, contentCommonInitJsp, false);
	}

	private void _createPageJSP(
			Component component, Map<String, Object> context)
		throws Exception {

		String pathName = component.getUncamelizedName(StringPool.UNDERLINE);
		String path = getJspOutputDir(component).concat(pathName);

		String contentJsp = _processTemplate(_tplJsp, context);
		String contentInitJsp = _processTemplate(_tplInitJsp, context);

		File initFile = new File(path.concat(_INIT_PAGE));
		File initExtFile = new File(path.concat(_INIT_EXT_PAGE));

		_writeFile(initFile, contentInitJsp);
		_writeFile(initExtFile, StringPool.BLANK, false);

		if (component.isBodyContent()) {
			String contentStart = _processTemplate(_tplStartJsp, context);

			File startFile = new File(path.concat(_START_PAGE));
			File endFile = new File(path.concat(_END_PAGE));

			_writeFile(startFile, contentStart, false);
			_writeFile(endFile, contentJsp, false);
		}
		else {
			File pageFile = new File(path.concat(_PAGE));

			_writeFile(pageFile, contentJsp, false);
		}
	}

	private void _createTag(
			Component component, Map<String, Object> context)
		throws Exception {

		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputDir(component));
		sb.append(component.getSafeName());
		sb.append(_CLASS_SUFFIX);

		String content = _processTemplate(_tplTag, context);

		File tagFile = new File(sb.toString());

		_writeFile(tagFile, content, false);
	}

	private void _createTld() throws Exception {
		Map<String, Object> context = getDefaultTemplateContext();

		List<Document> documents = new ArrayList<Document>();
		documents.add(_componentsDoc);
		documents.addAll(_componentsExtDoc);

		for (Document doc : documents) {
			Element root = doc.getRootElement();
			String shortName = GetterUtil.getString(
				root.attributeValue("short-name"), _DEFAULT_TAGLIB_SHORT_NAME);

			String uri = GetterUtil.getString(
				root.attributeValue("uri"), _DEFAULT_TAGLIB_URI);

			String version = GetterUtil.getString(
				root.attributeValue("tlib-version"), _DEFAULT_TAGLIB_VERSION);

			context.put("alloyComponent", shortName.equals(_DEFAULT_NAMESPACE));
			context.put("shortName", shortName);
			context.put("uri", uri);
			context.put("version", version);
			context.put("components", _getComponents(doc));

			String content = _processTemplate(_tplTld, context);

			String tldFilePath = _tldDir.concat(
				shortName).concat(_TLD_EXTENSION);

			File tldFile = new File(tldFilePath);

			Document outputDoc = SAXReaderUtil.read(content);

			if (tldFile.exists()) {
				outputDoc = _mergeTlds(SAXReaderUtil.read(tldFile), outputDoc);
			}

			_writeFile(tldFile, outputDoc.asXML());
		}
	}

	private List<Component> _getAllComponents() throws Exception {
		Document doc = SAXReaderUtil.createDocument();
		Element root = _componentsDoc.getRootElement().createCopy();

		for (Document extDoc : _componentsExtDoc) {
			Document componentsDoc = extDoc;

			Document parentDoc = _getComponentsDoc(
				extDoc.getRootElement().attributeValue("extends"));

			if (parentDoc != null) {
				componentsDoc = _mergeAttributes(extDoc, parentDoc);
			}

			List<Element> extComponents =
				componentsDoc.getRootElement().elements("component");

			for (Element extComponent : extComponents) {
				root.add(extComponent.createCopy());
			}
		}

		doc.add(root);

		return _getComponents(doc);
	}

	private List<Attribute> _getAttributes(Element componentNode) {
		return _getAttributes(componentNode, "attributes", "attribute");
	}

	private List<Attribute> _getAttributes(
		Element componentNode, String group, String nodeName) {

		List<Element> nodes = Collections.EMPTY_LIST;

		List<Attribute> attributes = new ArrayList<Attribute>();

		Element node = componentNode.element(group);

		if (node != null) {
			nodes = node.elements(nodeName);
		}

		for (Element attributeNode : nodes) {
			String name = attributeNode.elementText("name");
			String type = GetterUtil.getString(
				attributeNode.elementText("type"), _DEFAULT_TYPE);

			String inputType = GetterUtil.getString(
				attributeNode.elementText("inputType"), type);

			String outputType = GetterUtil.getString(
				attributeNode.elementText("outputType"), type);

			String defaultValue = attributeNode.elementText("defaultValue");
			String description = attributeNode.elementText("description");
			boolean required = GetterUtil.getBoolean(
				attributeNode.elementText("required"));

			attributes.add(
				new Attribute(name, inputType, outputType, defaultValue,
				description, required));
		}

		return attributes;
	}

	private Element _getComponentNode(Document doc, String name) {
		List<Element> components = doc.getRootElement().elements(_COMPONENT);

		for (Element component : components) {
			if (component.attributeValue("name").equals(name)) {
				return component;
			}
		}

		return null;
	}

	private List<Component> _getComponents(Document doc) throws Exception {
		List<Component> components = new ArrayList<Component>();
		Element root = doc.getRootElement();

		List<Element> allComponentNodes = root.elements("component");

		for (Element node : allComponentNodes) {
			String defaultPackage = root.attributeValue("short-name");

			String componentPackage = GetterUtil.getString(
				node.attributeValue("package"), defaultPackage);

			String name = node.attributeValue("name");
<commit_msg>Alloy Taglibs Set package on each extNode to not inherit the shot-name from the original document<commit_after>package com.liferay.alloy.tools.tagbuilder;

import com.liferay.alloy.tools.model.Attribute;
import com.liferay.alloy.tools.model.Component;
import com.liferay.portal.freemarker.FreeMarkerUtil;
import com.liferay.portal.kernel.util.FileUtil;
import com.liferay.portal.kernel.util.GetterUtil;
import com.liferay.portal.kernel.util.StringPool;
import com.liferay.portal.kernel.util.StringUtil;
import com.liferay.portal.kernel.xml.Document;
import com.liferay.portal.kernel.xml.Element;
import com.liferay.portal.kernel.xml.SAXReaderUtil;
import com.liferay.portal.util.FileImpl;
import com.liferay.portal.xml.SAXReaderImpl;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * <a href="TagBuilder.java.html"><b><i>View Source</i></b></a>
 *
 * @author <NAME>
 * @author <NAME>
 */
public class TagBuilder {

	public TagBuilder(
			String componentsXML, String componentsExtXML, String templatesDir,
			String javaOutputDir, String javaOutputPackage,	String jspDir,
			String jspOutputDir, String jspCommonInitPath,
			String jspCommonInitOutputDir, String tldDir)
		throws Exception {

		if (SAXReaderUtil.getSAXReader() == null) {
			(new SAXReaderUtil()).setSAXReader(new SAXReaderImpl());
		}

		_componentsExtXML = Arrays.asList(StringUtil.split(componentsExtXML));
		_componentsXML = componentsXML;
		_templatesDir = templatesDir;
		_javaOutputDir = javaOutputDir;
		_javaOutputPackage = javaOutputPackage;
		_jspDir = jspDir;
		_jspOutputDir = jspOutputDir;
		_jspCommonInitPath = jspCommonInitPath;
		_jspCommonInitOutputDir = jspCommonInitOutputDir;
		_tldDir = tldDir;

		_tplTld = _templatesDir + "tld.ftl";
		_tplTag = _templatesDir + "tag.ftl";
		_tplTagBase = _templatesDir + "tag_base.ftl";
		_tplCommonInitJsp = _templatesDir + "common_init_jsp.ftl";
		_tplJsp = _templatesDir + "jsp.ftl";
		_tplInitJsp = _templatesDir + "init_jsp.ftl";
		_tplStartJsp = _templatesDir + "start_jsp.ftl";

		_componentsExtDoc = new ArrayList<Document>();

		_componentsDoc = SAXReaderUtil.read(new File(_componentsXML));

		for (String componentExtXML : _componentsExtXML) {
			File extFile = new File(componentExtXML);

			if (extFile.exists()) {
				_componentsExtDoc.add(SAXReaderUtil.read(extFile));
			}
		}

		_create();
	}

	public static void main(String[] args) throws Exception {
		String componentsXML = System.getProperty("tagbuilder.components.xml");
		String componentsExtXML = System.getProperty("tagbuilder.components.ext.xml");
		String templatesDir = System.getProperty("tagbuilder.templates.dir");
		String javaOutputDir = System.getProperty("tagbuilder.java.output.dir");
		String javaOutputPackage = System.getProperty("tagbuilder.java.output.package");
		String jspDir = System.getProperty("tagbuilder.jsp.dir");
		String jspCommonInitOutputDir = System.getProperty("tagbuilder.jsp.common.init.output.dir");
		String jspCommonInitPath = System.getProperty("tagbuilder.jsp.common.init.path");
		String jspOutputDir = System.getProperty("tagbuilder.jsp.output.dir");
		String tldDir = System.getProperty("tagbuilder.tld.dir");

		new TagBuilder(
			componentsXML, componentsExtXML, templatesDir, javaOutputDir,
			javaOutputPackage, jspDir, jspOutputDir, jspCommonInitPath,
			jspCommonInitOutputDir,  tldDir);
	}

	public Map<String, Object> getDefaultTemplateContext() {
		Map<String, Object> context = new HashMap<String, Object>();

		context.put("authors", AUTHORS);
		context.put("jspCommonInitPath", _jspCommonInitPath);
		context.put("jspDir", _jspDir);
		context.put("packagePath", _javaOutputPackage);

		return context;
	}

	public String getJavaOutputBaseDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputDir(component));
		sb.append(_BASE);
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJavaOutputDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_javaOutputDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJspDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_jspDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public String getJspOutputDir(Component component) {
		StringBuilder sb = new StringBuilder();

		sb.append(_jspOutputDir);
		sb.append(component.getPackage());
		sb.append(StringPool.SLASH);

		return sb.toString();
	}

	public Map<String, Object> getTemplateContext(Component component) {
		Map<String, Object> context = getDefaultTemplateContext();

		String jspRelativePath = getJspDir(component).concat(
			component.getUncamelizedName(StringPool.UNDERLINE));

		context.put("component", component);
		context.put("namespace", component.getAttributeNamespace());
		context.put("jspRelativePath", jspRelativePath);

		return context;
	}

	private void _create() throws Exception {
		List<Component> components = _getAllComponents();

		for (Component component : components) {
			Map<String, Object> context = getTemplateContext(component);

			_createBaseTag(component, context);
			_createPageJSP(component, context);
			_createTag(component, context);
		}

		_createCommonInitJSP();
		_createTld();
	}

	private void _createBaseTag(
			Component component, Map<String, Object> context)
		throws Exception {

		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputBaseDir(component));
		sb.append(_BASE_CLASS_PREFIX);
		sb.append(component.getSafeName());
		sb.append(_CLASS_SUFFIX);

		String content = _processTemplate(_tplTagBase, context);

		File tagFile = new File(sb.toString());

		_writeFile(tagFile, content);
	}

	private void _createCommonInitJSP() throws Exception {
		Map<String, Object> context = getDefaultTemplateContext();

		String contentCommonInitJsp = _processTemplate(_tplCommonInitJsp, context);

		File commonInitFile = new File(_jspCommonInitOutputDir);
		_writeFile(commonInitFile, contentCommonInitJsp, false);
	}

	private void _createPageJSP(
			Component component, Map<String, Object> context)
		throws Exception {

		String pathName = component.getUncamelizedName(StringPool.UNDERLINE);
		String path = getJspOutputDir(component).concat(pathName);

		String contentJsp = _processTemplate(_tplJsp, context);
		String contentInitJsp = _processTemplate(_tplInitJsp, context);

		File initFile = new File(path.concat(_INIT_PAGE));
		File initExtFile = new File(path.concat(_INIT_EXT_PAGE));

		_writeFile(initFile, contentInitJsp);
		_writeFile(initExtFile, StringPool.BLANK, false);

		if (component.isBodyContent()) {
			String contentStart = _processTemplate(_tplStartJsp, context);

			File startFile = new File(path.concat(_START_PAGE));
			File endFile = new File(path.concat(_END_PAGE));

			_writeFile(startFile, contentStart, false);
			_writeFile(endFile, contentJsp, false);
		}
		else {
			File pageFile = new File(path.concat(_PAGE));

			_writeFile(pageFile, contentJsp, false);
		}
	}

	private void _createTag(
			Component component, Map<String, Object> context)
		throws Exception {

		StringBuilder sb = new StringBuilder();

		sb.append(getJavaOutputDir(component));
		sb.append(component.getSafeName());
		sb.append(_CLASS_SUFFIX);

		String content = _processTemplate(_tplTag, context);

		File tagFile = new File(sb.toString());

		_writeFile(tagFile, content, false);
	}

	private void _createTld() throws Exception {
		Map<String, Object> context = getDefaultTemplateContext();

		List<Document> documents = new ArrayList<Document>();
		documents.add(_componentsDoc);
		documents.addAll(_componentsExtDoc);

		for (Document doc : documents) {
			Element root = doc.getRootElement();
			String shortName = GetterUtil.getString(
				root.attributeValue("short-name"), _DEFAULT_TAGLIB_SHORT_NAME);

			String uri = GetterUtil.getString(
				root.attributeValue("uri"), _DEFAULT_TAGLIB_URI);

			String version = GetterUtil.getString(
				root.attributeValue("tlib-version"), _DEFAULT_TAGLIB_VERSION);

			context.put("alloyComponent", shortName.equals(_DEFAULT_NAMESPACE));
			context.put("shortName", shortName);
			context.put("uri", uri);
			context.put("version", version);
			context.put("components", _getComponents(doc));

			String content = _processTemplate(_tplTld, context);

			String tldFilePath = _tldDir.concat(
				shortName).concat(_TLD_EXTENSION);

			File tldFile = new File(tldFilePath);

			Document outputDoc = SAXReaderUtil.read(content);

			if (tldFile.exists()) {
				outputDoc = _mergeTlds(SAXReaderUtil.read(tldFile), outputDoc);
			}

			_writeFile(tldFile, outputDoc.asXML());
		}
	}

	private List<Component> _getAllComponents() throws Exception {
		Document doc = SAXReaderUtil.createDocument();
		Element root = _componentsDoc.getRootElement().createCopy();

		for (Document extDoc : _componentsExtDoc) {
			Document componentsDoc = extDoc;
			Element extRoot = componentsDoc.getRootElement();

			String extDefaultPackage = extRoot.attributeValue("short-name");
			List<Element> allExtComponentNodes = extRoot.elements("component");

			// Set package on each extNode to not inherit the shot-name from the original document
			for (Element extNode : allExtComponentNodes) {
				String extComponentPackage = GetterUtil.getString(
					extNode.attributeValue("package"), extDefaultPackage);

				extNode.addAttribute("package", extComponentPackage);
			}

			Document parentDoc = _getComponentsDoc(
				extDoc.getRootElement().attributeValue("extends"));

			if (parentDoc != null) {
				componentsDoc = _mergeAttributes(extDoc, parentDoc);
			}

			List<Element> extComponents =
				componentsDoc.getRootElement().elements("component");

			for (Element extComponent : extComponents) {
				root.add(extComponent.createCopy());
			}
		}

		doc.add(root);

		return _getComponents(doc);
	}

	private List<Attribute> _getAttributes(Element componentNode) {
		return _getAttributes(componentNode, "attributes", "attribute");
	}

	private List<Attribute> _getAttributes(
		Element componentNode, String group, String nodeName) {

		List<Element> nodes = Collections.EMPTY_LIST;

		List<Attribute> attributes = new ArrayList<Attribute>();

		Element node = componentNode.element(group);

		if (node != null) {
			nodes = node.elements(nodeName);
		}

		for (Element attributeNode : nodes) {
			String name = attributeNode.elementText("name");
			String type = GetterUtil.getString(
				attributeNode.elementText("type"), _DEFAULT_TYPE);

			String inputType = GetterUtil.getString(
				attributeNode.elementText("inputType"), type);

			String outputType = GetterUtil.getString(
				attributeNode.elementText("outputType"), type);

			String defaultValue = attributeNode.elementText("defaultValue");
			String description = attributeNode.elementText("description");
			boolean required = GetterUtil.getBoolean(
				attributeNode.elementText("required"));

			attributes.add(
				new Attribute(name, inputType, outputType, defaultValue,
				description, required));
		}

		return attributes;
	}

	private Element _getComponentNode(Document doc, String name) {
		List<Element> components = doc.getRootElement().elements(_COMPONENT);

		for (Element component : components) {
			if (component.attributeValue("name").equals(name)) {
				return component;
			}
		}

		return null;
	}

	private List<Component> _getComponents(Document doc) throws Exception {
		List<Component> components = new ArrayList<Component>();
		Element root = doc.getRootElement();

		String defaultPackage = root.attributeValue("short-name");
		List<Element> allComponentNodes = root.elements("component");

		for (Element node : allComponentNodes) {
			String componentPackage = GetterUtil.getString(
				node.attributeValue("package"), defaultPackage);

			String name = node.attributeValue("name");

{'fraction_non_alphanumeric': 0.11482663263135504, 'fraction_numerical': 0.0, 'mean_word_length': 5.361569688768606, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '537565', 'n_tokens_mistral': 7725, 'n_tokens_neox': 6994, 'n_words': 1409}