<commit_before># This file is part of NIT ( http://www.nitlanguage.org ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Basic catalog generator for Nit packages
#
# See: <http://nitlanguage.org/catalog/>
#
# The tool scans packages and generates the HTML files of a catalog.
#
# ## Features
#
# * [X] scan packages and their `.ini`
# * [X] generate lists of packages
# * [X] generate a page per package with the readme and most metadata
# * [ ] link/include/be included in the documentation
# * [ ] propose `related packages`
# * [X] show directory content (a la nitls)
# * [X] gather git information from the working directory
# * [ ] gather git information from the repository
# * [ ] gather package information from github
# * [ ] gather people information from github
# * [ ] reify people
# * [ ] separate information gathering from rendering
# * [ ] move up information gathering in (existing or new) service modules
# * [X] add command line options
# * [ ] harden HTML (escaping, path injection, etc)
# * [ ] nitcorn server with RESTful API
#
# ## Issues and limitations
#
# The tool works likee the other tools and expects to find valid Nit source code in the directories
#
# * cruft and temporary files will be collected
# * missing source file (e.g. not yet generated by nitcc) will make information
#   incomplete (e.g. invalid module thus partial dependency and metrics)
#
# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
module nitcatalog

import loader # Scan&load packages, groups and modules
import doc::doc_down # Display mdoc
import md5 # To get gravatar images
import counter # For statistics
import modelize # To process and count classes and methods

redef class MPackage
	# Return the associated metadata from the `ini`, if any
	fun metadata(key: String): nullable String
	do
		var ini = self.ini
		if ini == null then return null
		return ini[key]
	end

	# The list of maintainers
	var maintainers = new Array[String]

	# The list of contributors
	var contributors = new Array[String]

	# The date of the most recent commit
	var last_date: nullable String = null

	# The date of the oldest commit
	var first_date: nullable String = null
end

# A HTML page in a catalog
#
# This is just a template with the header pre-filled and the footer injected at rendering.
# Therefore, once instantiated, the content can just be added to it.
class CatalogPage
	super Template

	# Placeholder to include additional things before the `</head>`.
	var more_head = new Template

	redef init
	do
		add """
<!DOCTYPE html>
<html>
<head>
	<meta charset="utf-8">
	<link rel="stylesheet" media="all" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
	<link rel="stylesheet" media="all" href="style.css">
"""
		add more_head

		add """
</head>
<body>
<div class='container-fluid'>
 <div class='row'>
  <nav id='topmenu' class='navbar navbar-default navbar-fixed-top' role='navigation'>
   <div class='container-fluid'>
    <div class='navbar-header'>
     <button type='button' class='navbar-toggle' data-toggle='collapse' data-target='#topmenu-collapse'>
      <span class='sr-only'>Toggle menu</span>
      <span class='icon-bar'></span>
      <span class='icon-bar'></span>
      <span class='icon-bar'></span>
     </button>
     <span class='navbar-brand'><a href="http://nitlanguage.org/">Nitlanguage.org</a></span>
    </div>
    <div class='collapse navbar-collapse' id='topmenu-collapse'>
     <ul class='nav navbar-nav'>
      <li><a href="index.html">Catalog</a></li>
     </ul>
    </div>
   </div>
  </nav>
 </div>
"""
	end

	redef fun rendering
	do
		add """
</div> <!-- container-fluid -->
<script src='https://code.jquery.com/jquery-latest.min.js'></script>
<script src='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js'></script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table-all.min.js'></script>
</body>
</html>
"""
	end
end

redef class Int
	# Returns `log(self+1)`. Used to compute score of packages
	fun score: Float do return (self+1).to_f.log
end

# The main class of the calatog generator that has the knowledge
class Catalog

	# The modelbuilder
	# used to access the files and count source lines of code
	var modelbuilder: ModelBuilder

	# Packages by tag
	var tag2proj = new MultiHashMap[String, MPackage]

	# Packages by category
	var cat2proj = new MultiHashMap[String, MPackage]

	# Packages by maintainer
	var maint2proj = new MultiHashMap[String, MPackage]

	# Packages by contributors
	var contrib2proj = new MultiHashMap[String, MPackage]

	# Dependency between packages
	var deps = new POSet[MPackage]

	# Number of modules by package
	var mmodules = new Counter[MPackage]

	# Number of classes by package
	var mclasses = new Counter[MPackage]

	# Number of methods by package
	var mmethods = new Counter[MPackage]

	# Number of line of code by package
	var loc = new Counter[MPackage]

	# Number of commits by package
	var commits = new Counter[MPackage]

	# Score by package
	#
	# The score is loosely computed using other metrics
	var score = new Counter[MPackage]

	# Scan, register and add a contributor to a package
	fun add_contrib(person: String, mpackage: MPackage, res: Template)
	do
		var projs = contrib2proj[person]
		if not projs.has(mpackage) then projs.add mpackage
		var name = person
		var email = null
		var page = null

		# Regular expressions are broken, need to investigate.
		# So split manually.
		#
		#var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
		#var m = (person+" ").search(re)
		#print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
		do
			var sp1 = person.split_once_on("<")
			if sp1.length < 2 then
				break
			end
			var sp2 = sp1.last.split_once_on(">")
			if sp2.length < 2 then
				break
			end
			name = sp1.first.trim
			email = sp2.first.trim
			var sp3 = sp2.last.split_once_on("(")
			if sp3.length < 2 then
				break
			end
			var sp4 = sp3.last.split_once_on(")")
			if sp4.length < 2 then
				break
			end
			page = sp4.first.trim
		end

		var e = name.html_escape
		res.add "<li>"
		if page != null then
			res.add "<a href=\"{page.html_escape}\">"
		end
		if email != null then
			# TODO get more things from github by using the email as a key
			# "https://api.github.com/search/users?q={email}+in:email"
			var md5 = email.md5.to_lower
			res.add "<img src=\"https://secure.gravatar.com/avatar/{md5}?size=20&amp;default=retro\">&nbsp;"
		end
		res.add "{e}"
		if page != null then res.add "</a>"
		res.add "</li>"
	end

	# Recursively generate a level in the file tree of the *content* section
	private fun gen_content_level(ot: OrderedTree[Object], os: Array[Object], res: Template)
	do
		res.add "<ul>\n"
		for o in os do
			res.add "<li>"
			if o isa MGroup then
				var d = ""
				var mdoc = o.mdoc
				if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
				res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
			else if o isa ModulePath then
				var d = ""
				var m = o.mmodule
				if m != null then
					var mdoc = m.mdoc
					if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
				end
				res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
			else
				abort
			end
			var subs = ot.sub.get_or_null(o)
			if subs != null then gen_content_level(ot, subs, res)
			res.add "</li>\n"
		end
		res.add "</ul>\n"
	end

	# Compute information and generate a full HTML page for a package
	fun package_page(mpackage: MPackage): Writable
	do
		var res = new CatalogPage
		var score = score[mpackage].to_f
		var name = mpackage.name.html_escape
		res.more_head.add """<title>{{{name}}}</title>"""

		res.add """
<div class="content">
<h1 class="package-name">{{{name}}}</h1>
"""
		var mdoc = mpackage.mdoc_or_fallback
		if mdoc != null then
			score += 100.0
			res.add mdoc.html_documentation
			score += mdoc.content.length.score
		end

		res.add "<h2>Content</h2>"
		var ot = new OrderedTree[Object]
		for g in mpackage.mgroups do
			var pa = g.parent
			if g.is_interesting then
				ot.add(pa, g)
				pa = g
			end
			for mp in g.module_paths do
				ot.add(pa, mp)
			end
		end
		ot.sort_with(alpha_comparator)
		gen_content_level(ot, ot.roots, res)


		res.add """
</div>
<div class="sidebar">
<ul class="box">
"""
		var homepage = mpackage.metadata("upstream.homepage")
		if homepage != null then
			score += 5.0
			var e = homepage.html_escape
			res.add "<li><a href=\"{e}\">{e}</a></li>\n"
		end
		var maintainer = mpackage.metadata("package.maintainer")
		if maintainer != null then
			score += 5.0
			add_contrib(maintainer, mpackage, res)
			mpackage.maintainers.add maintainer
			var projs = maint2proj[maintainer]
			if not projs.has(mpackage) then projs.add mpackage
		end
		var license = mpackage.metadata("package.license")
		if license != null then
			score += 5.0
			var e = license.html_escape
			res.add "<li><a href=\"http://opensource.org/licenses/{e}\">{e}</a> license</li>\n"
		end
		res.add "</ul>\n"

		res.add "<h3>Source Code</h3>\n<ul class=\"box\">\n"
		var browse = mpackage.metadata("upstream.browse")
		if browse != null then
			score += 5.0
			var e = browse.html_escape
			res.add "<li><a href=\"{e}\">{e}</a></li>\n"
		end
		var git = mpackage.metadata("upstream.git")
		if git != null then
			var e = git.html_escape
			res.add "<li><tt>{e}</tt></li>\n"
		end
		var last_date = mpackage.last_date
		if last_date != null then
			var e = last_date.html_escape
			res.add "<li>most recent commit: {e}</li>\n"
		end
		var first_date = mpackage.first_date
		if first_date != null then
			var e = first_date.html_escape
			res.add "<li>oldest commit: {e}</li>\n"
		end
		var commits = commits[mpackage]
		if commits != 0 then
			res.add "<li>{commits} commits</li>\n"
		end
		res.add "</ul>\n"

		res.add "<h3>Tags</h3>\n"
		var tags = mpackage.metadata("package.tags")
		var ts2 = new Array[String]
		var cat = null
		if tags != null then
			var ts = tags.split(",")
			for t in ts do
				t = t.trim
				if t == "" then continue
				if cat == null then cat = t
				tag2proj[t].add mpackage
				t = t.html_escape
				ts2.add "<a href=\"index.html#tag_{t}\">{t}</a>"
			end
			res.add_list(ts2, ", ", ", ")
		end
		if ts2.is_empty then
			var t = "none"
			cat = t
			tag2proj[t].add mpackage
			res.add "<a href=\"index.html#tag_{t}\">{t}</a>"
		end
		if cat != null then cat2proj[cat].add mpackage
		score += ts2.length.score

		var reqs = deps[mpackage].greaters.to_a
		reqs.remove(mpackage)
		alpha_comparator.sort(reqs)
		res.add "<h3>Requirements</h3>\n"
		if reqs.is_empty then
			res.add "none"
		else
			var list = new Array[String]
			for r in reqs do
				var direct = deps.has_direct_edge(mpackage, r)
				var s = "<a href=\"{r}.html\">"
				if direct then s += "<strong>"
				s += r.to_s
				if direct then s += "</strong>"
				s += "</a>"
				list.add s
			end
			res.add_list(list, ", ", " and ")
		end

		reqs = deps[mpackage].smallers.to_a
		reqs.remove(mpackage)
		alpha_comparator.sort(reqs)
		res.add "<h3>Clients</h3>\n"
		if reqs.is_empty then
			res.add "none"
		else
			var list = new Array[String]
			for r in reqs do
				var direct = deps.has_direct_edge(r, mpackage)
				var s = "<a href=\"{r}.html\">"
				if direct then s += "<strong>"
				s += r.to_s
				if direct then s += "</strong>"
				s += "</a>"
				list.add s
			end
			res.add_list(list, ", ", " and ")
		end

		score += deps[mpackage].greaters.length.score
		score += deps[mpackage].direct_greaters.length.score
		score += deps[mpackage].smallers.length.score
		score += deps[mpackage].direct_smallers.length.score

		var contributors = mpackage.contributors
		if not contributors.is_empty then
			res.add "<h3>Contributors</h3>\n<ul class=\"box\">"
			for c in contributors do
				add_contrib(c, mpackage, res)
			end
			res.add "</ul>"
		end
		score += contributors.length.to_f

		var mmodules = 0
		var mclasses = 0
		var mmethods = 0
		var loc = 0
		for g in mpackage.mgroups do
			mmodules += g.module_paths.length
			for m in g.mmodules do
				var am = modelbuilder.mmodule2node(m)
				if am != null then
					var file = am.location.file
					if file != null then
						loc += file.line_starts.length - 1
					end
				end
				for cd in m.mclassdefs do
					mclasses += 1
					for pd in cd.mpropdefs do
						if not pd isa MMethodDef then continue
						mmethods += 1
					end
				end
			end
		end
		self.mmodules[mpackage] = mmodules
		self.mclasses[mpackage] = mclasses
		self.mmethods[mpackage] = mmethods
		self.loc[mpackage] = loc

		#score += mmodules.score
		score += mclasses.score
		score += mmethods.score
		score += loc.score

		res.add """
<h3>Stats</h3>
<ul class="box">
<li>{{{mmodules}}} modules</li>
<li>{{{mclasses}}} classes</li>
<li>{{{mmethods}}} methods</li>
<li>{{{loc}}} lines of code</li>
</ul>
"""

		res.add """
</div>
"""
		self.score[mpackage] = score.to_i

		return res
	end

	# Return a short HTML sequence for a package
	#
	# Intended to use in lists.
	fun li_package(p: MPackage): String
	do
		var res = ""
		var f = "{p.name}.html"
		res += "<a href=\"{f}\">{p}</a>"
		var d = p.mdoc_or_fallback
		if d != null then res += " - {d.html_synopsis.write_to_string}"
		return res
	end

	# List packages by group.
	#
	# For each key of the `map` a `<h3>` is generated.
	# Each package is then listed.
	#
	# The list of keys is generated first to allow fast access to the correct `<h3>`.
	# `id_prefix` is used to give an id to the `<h3>` element.
	fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template
	do
		var res = new Template
		var keys = map.keys.to_a
		alpha_comparator.sort(keys)
		var list = [for x in keys do "<a href=\"#{id_prefix}{x.html_escape}\">{x.html_escape}</a>"]
		res.add_list(list, ", ", " and ")

		for k in keys do
			var projs = map[k].to_a
			alpha_comparator.sort(projs)
			var e = k.html_escape
			res.add "<h3 id=\"{id_prefix}{e}\">{e} ({projs.length})</h3>\n<ul>\n"
			for p in projs do
				res.add "<li>"
				res.add li_package(p)
				res.add "</li>"
			end
			res.add "</ul>"
		end
		return res
	end

	# List the 10 best packages from `cpt`
	fun list_best(cpt: Counter[MPackage]): Template
	do
		var res = new Template
		res.add "<ul>"
		var best = cpt.sort
		for i in [1..10] do
			if i > best.length then break
			var p = best[best.length-i]
			res.add "<li>"
			res.add li_package(p)
			# res.add " ({cpt[p]})"
			res.add "</li>"
		end
		res.add "</ul>"
		return res
	end

	# Collect more information on a package using the `git` tool.
	fun git_info(mpackage: MPackage)
	do
		var ini = mpackage.ini
		if ini == null then return

		# TODO use real git info
		#var repo = ini.get_or_null("upstream.git")
		#var branch = ini.get_or_null("upstream.git.branch")
		#var directory = ini.get_or_null("upstream.git.directory")

		var dirpath = mpackage.root.filepath
		if dirpath == null then return

		# Collect commits info
		var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
		var contributors = new Counter[String]
		var commits = res.split("\n")
		if commits.not_empty and commits.last == "" then commits.pop
		self.commits[mpackage] = commits.length
		for l in commits do
			var s = l.split_once_on(';')
			if s.length != 2 or s.last == "" then continue

			# Collect date of last and first commit
			if mpackage.last_date == null then mpackage.last_date = s.first
			mpackage.first_date = s.first

			# Count contributors
			contributors.inc(s.last)
		end
		for c in contributors.sort.reverse_iterator do
			mpackage.contributors.add c
		end

	end

	# Produce a HTML table containig information on the packages
	#
	# `package_page` must have been called before so that information is computed.
	fun table_packages(mpackages: Array[MPackage]): Template
	do
		alpha_comparator.sort(mpackages)
		var res = new Template
		res.add "<table data-toggle=\"table\" data-sort-name=\"name\" data-sort-order=\"desc\" width=\"100%\">\n"
		res.add "<thead><tr>\n"
		res.add "<th data-field=\"name\" data-sortable=\"true\">name</th>\n"
		res.add "<th data-field=\"maint\" data-sortable=\"true\">maint</th>\n"
		res.add "<th data-field=\"contrib\" data-sortable=\"true\">contrib</th>\n"
		res.add "<th data-field=\"reqs\" data-sortable=\"true\">reqs</th>\n"
		res.add "<th data-field=\"dreqs\" data-sortable=\"true\">direct<br>reqs</th>\n"
		res.add "<th data-field=\"cli\" data-sortable=\"true\">clients</th>\n"
		res.add "<th data-field=\"dcli\" data-sortable=\"true\">direct<br>clients</th>\n"
		res.add "<th data-field=\"mod\" data-sortable=\"true\">modules</th>\n"
		res.add "<th data-field=\"cla\" data-sortable=\"true\">classes</th>\n"
		res.add "<th data-field=\"met\" data-sortable=\"true\">methods</th>\n"
		res.add "<th data-field=\"loc\" data-sortable=\"true\">lines</th>\n"
		res.add "<th data-field=\"score\" data-sortable=\"true\">score</th>\n"
		res.add "</tr></thead>"
		for p in mpackages do
			res.add "<tr>"
			res.add "<td><a href=\"{p.name}.html\">{p.name}</a></td>"
			var maint = "?"
			if p.maintainers.not_empty then maint = p.maintainers.first
			res.add "<td>{maint}</td>"
			res.add "<td>{p.contributors.length}</td>"
			res.add "<td>{deps[p].greaters.length-1}</td>"
			res.add "<td>{deps[p].direct_greaters.length}</td>"
			res.add "<td>{deps[p].smallers.length-1}</td>"
			res.add "<td>{deps[p].direct_smallers.length}</td>"
			res.add "<td>{mmodules[p]}</td>"
			res.add "<td>{mclasses[p]}</td>"
			res.add "<td>{mmethods[p]}</td>"
			res.add "<td>{loc[p]}</td>"
			res.add "<td>{score[p]}</td>"
			res.add "</tr>\n"
		end
		res.add "</table>\n"
		return res
	end
end

# Execute a git command and return the result
fun git_run(command: String...): String
do
	# print "git {command.join(" ")}"
	var p = new ProcessReader("git", command...)
	var res = p.read_all
	p.close
	p.wait
	return res
end

var model = new Model
var tc = new ToolContext

var opt_dir = new OptionString("Directory where the HTML files are generated", "-d", "--dir")
var opt_no_git = new OptionBool("Do not gather git information from the working directory", "--no-git")
var opt_no_parse = new OptionBool("Do not parse nit files (no importation information)", "--no-parse")
var opt_no_model = new OptionBool("Do not analyse nit files (no class/method information)", "--no-model")

tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model)

tc.process_options(sys.args)
tc.keep_going = true

var modelbuilder = new ModelBuilder(model, tc)
var catalog = new Catalog(modelbuilder)

# Get files or groups
for a in tc.option_context.rest do
	modelbuilder.get_mgroup(a)
	modelbuilder.identify_file(a)
end

# Scan packages and compute information
for p in model.mpackages do
	var g = p.root
	assert g != null
	modelbuilder.scan_group(g)

	# Load the module to process importation information
	if opt_no_parse.value then continue
	modelbuilder.parse_group(g)

	catalog.deps.add_node(p)
	for gg in p.mgroups do for m in gg.mmodules do
		for im in m.in_importation.direct_greaters do
			var ip = im.mpackage
			if ip == null or ip == p then continue
			catalog.deps.add_edge(p, ip)
		end
	end
end

if not opt_no_git.value then for p in model.mprojects do
	catalog.git_info(p)
end

# Run phases to modelize classes and properties (so we can count them)
if not opt_no_model.value then
	modelbuilder.run_phases
end

var out = opt_dir.value or else "catalog.out"
out.mkdir

# Generate the css (hard coded)
var css = """
body {
margin-top:			15px;
background-color:		#f8f8f8;
}

a {
color:				#0D8921;
text-decoration:		none;
}

a:hover {
color:				#333;
text-decoration:		none;
}

h1 {
font-weight:			bold;
color:                          #0D8921;
font-size:			22px;
}

h2 {
color:                          #6C6C6C;
font-size:			18px;
border-bottom:			solid 3px #CCC;
}

h3 {
color:                          #6C6C6C;
font-size:			15px;
border-bottom:			solid 1px #CCC;
}

ul {
list-style-type:		square;
}

dd {
color:				#6C6C6C;
margin-top:			1em;
margin-bottom:			1em;
}

pre {
border:				1px solid #CCC;
font-family:			Monospace;
color:				#2d5003;
background-color:		rgb(250, 250, 250);
}

code {
font-family:			Monospace;
color:				#2d5003;
}

footer {
margin-top:			20px;
}

.container {
margin:				0 auto;
padding:			0 20px;
}

.content {
float:				left;
margin-top:			40px;
width:				65%;
}

.sidebar {
float:				right;
margin-top:			40px;
width:				30%
}

.sidebar h3 {
color:                          #0D8921;
font-size:			18px;
border-bottom:			0px;
}

.box {
margin:				0;
padding:			0;
}

.box li {
line-height:			2.5;
white-space:			nowrap;
overflow:			hidden;
text-overflow:			ellipsis;
padding-right:			10px;
border-bottom:			1px solid rgba(0,0,0,0.2);
}
"""
css.write_to_file(out/"style.css")

# PAGES

for p in model.mpackages do
	# print p
	var f = "{p.name}.html"
	catalog.package_page(p).write_to_file(out/f)
end

# INDEX

var index = new CatalogPage
index.more_head.add "<title>Packages in Nit</title>"

index.add """
<div class="content">
<h1>Packages in Nit</h1>
"""

index.add "<h2>Highlighted Packages</h2>\n"
index.add catalog.list_best(catalog.score)

index.add "<h2>Most Required</h2>\n"
var reqs = new Counter[MPackage]
for p in model.mpackages do
	reqs[p] = catalog.deps[p].smallers.length - 1
end
index.add catalog.list_best(reqs)

index.add "<h2>By First Tag</h2>\n"
index.add catalog.list_by(catalog.cat2proj, "cat_")

index.add "<h2>By Any Tag</h2>\n"
index.add catalog.list_by(catalog.tag2proj, "tag_")

index.add """
</div>
<div class="sidebar">
<h3>Stats</h3>
<ul class="box">
<li>{{{model.mpackages.length}}} packages</li>
<li>{{{catalog.maint2proj.length}}} maintainers</li>
<li>{{{catalog.contrib2proj.length}}} contributors</li>
<li>{{{catalog.tag2proj.length}}} tags</li>
<li>{{{catalog.mmodules.sum}}} modules</li>
<li>{{{catalog.mclasses.sum}}} classes</li>
<li>{{{catalog.mmethods.sum}}} methods</li>
<li>{{{catalog.loc.sum}}} lines of code</li>
</ul>
</div>
"""

index.write_to_file(out/"index.html")

# PEOPLE<commit_msg>nitcatalog: do not crash if the dependency poset is empty (--no-parse)<commit_after># This file is part of NIT ( http://www.nitlanguage.org ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Basic catalog generator for Nit packages
#
# See: <http://nitlanguage.org/catalog/>
#
# The tool scans packages and generates the HTML files of a catalog.
#
# ## Features
#
# * [X] scan packages and their `.ini`
# * [X] generate lists of packages
# * [X] generate a page per package with the readme and most metadata
# * [ ] link/include/be included in the documentation
# * [ ] propose `related packages`
# * [X] show directory content (a la nitls)
# * [X] gather git information from the working directory
# * [ ] gather git information from the repository
# * [ ] gather package information from github
# * [ ] gather people information from github
# * [ ] reify people
# * [ ] separate information gathering from rendering
# * [ ] move up information gathering in (existing or new) service modules
# * [X] add command line options
# * [ ] harden HTML (escaping, path injection, etc)
# * [ ] nitcorn server with RESTful API
#
# ## Issues and limitations
#
# The tool works likee the other tools and expects to find valid Nit source code in the directories
#
# * cruft and temporary files will be collected
# * missing source file (e.g. not yet generated by nitcc) will make information
#   incomplete (e.g. invalid module thus partial dependency and metrics)
#
# How to use the tool as the basis of a Nit code archive on the web usable with a package manager is not clear.
module nitcatalog

import loader # Scan&load packages, groups and modules
import doc::doc_down # Display mdoc
import md5 # To get gravatar images
import counter # For statistics
import modelize # To process and count classes and methods

redef class MPackage
	# Return the associated metadata from the `ini`, if any
	fun metadata(key: String): nullable String
	do
		var ini = self.ini
		if ini == null then return null
		return ini[key]
	end

	# The list of maintainers
	var maintainers = new Array[String]

	# The list of contributors
	var contributors = new Array[String]

	# The date of the most recent commit
	var last_date: nullable String = null

	# The date of the oldest commit
	var first_date: nullable String = null
end

# A HTML page in a catalog
#
# This is just a template with the header pre-filled and the footer injected at rendering.
# Therefore, once instantiated, the content can just be added to it.
class CatalogPage
	super Template

	# Placeholder to include additional things before the `</head>`.
	var more_head = new Template

	redef init
	do
		add """
<!DOCTYPE html>
<html>
<head>
	<meta charset="utf-8">
	<link rel="stylesheet" media="all" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">
	<link rel="stylesheet" media="all" href="style.css">
"""
		add more_head

		add """
</head>
<body>
<div class='container-fluid'>
 <div class='row'>
  <nav id='topmenu' class='navbar navbar-default navbar-fixed-top' role='navigation'>
   <div class='container-fluid'>
    <div class='navbar-header'>
     <button type='button' class='navbar-toggle' data-toggle='collapse' data-target='#topmenu-collapse'>
      <span class='sr-only'>Toggle menu</span>
      <span class='icon-bar'></span>
      <span class='icon-bar'></span>
      <span class='icon-bar'></span>
     </button>
     <span class='navbar-brand'><a href="http://nitlanguage.org/">Nitlanguage.org</a></span>
    </div>
    <div class='collapse navbar-collapse' id='topmenu-collapse'>
     <ul class='nav navbar-nav'>
      <li><a href="index.html">Catalog</a></li>
     </ul>
    </div>
   </div>
  </nav>
 </div>
"""
	end

	redef fun rendering
	do
		add """
</div> <!-- container-fluid -->
<script src='https://code.jquery.com/jquery-latest.min.js'></script>
<script src='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js'></script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table-all.min.js'></script>
</body>
</html>
"""
	end
end

redef class Int
	# Returns `log(self+1)`. Used to compute score of packages
	fun score: Float do return (self+1).to_f.log
end

# The main class of the calatog generator that has the knowledge
class Catalog

	# The modelbuilder
	# used to access the files and count source lines of code
	var modelbuilder: ModelBuilder

	# Packages by tag
	var tag2proj = new MultiHashMap[String, MPackage]

	# Packages by category
	var cat2proj = new MultiHashMap[String, MPackage]

	# Packages by maintainer
	var maint2proj = new MultiHashMap[String, MPackage]

	# Packages by contributors
	var contrib2proj = new MultiHashMap[String, MPackage]

	# Dependency between packages
	var deps = new POSet[MPackage]

	# Number of modules by package
	var mmodules = new Counter[MPackage]

	# Number of classes by package
	var mclasses = new Counter[MPackage]

	# Number of methods by package
	var mmethods = new Counter[MPackage]

	# Number of line of code by package
	var loc = new Counter[MPackage]

	# Number of commits by package
	var commits = new Counter[MPackage]

	# Score by package
	#
	# The score is loosely computed using other metrics
	var score = new Counter[MPackage]

	# Scan, register and add a contributor to a package
	fun add_contrib(person: String, mpackage: MPackage, res: Template)
	do
		var projs = contrib2proj[person]
		if not projs.has(mpackage) then projs.add mpackage
		var name = person
		var email = null
		var page = null

		# Regular expressions are broken, need to investigate.
		# So split manually.
		#
		#var re = "([^<(]*?)(<([^>]*?)>)?(\\((.*)\\))?".to_re
		#var m = (person+" ").search(re)
		#print "{person}: `{m or else "?"}` `{m[1] or else "?"}` `{m[3] or else "?"}` `{m[5] or else "?"}`"
		do
			var sp1 = person.split_once_on("<")
			if sp1.length < 2 then
				break
			end
			var sp2 = sp1.last.split_once_on(">")
			if sp2.length < 2 then
				break
			end
			name = sp1.first.trim
			email = sp2.first.trim
			var sp3 = sp2.last.split_once_on("(")
			if sp3.length < 2 then
				break
			end
			var sp4 = sp3.last.split_once_on(")")
			if sp4.length < 2 then
				break
			end
			page = sp4.first.trim
		end

		var e = name.html_escape
		res.add "<li>"
		if page != null then
			res.add "<a href=\"{page.html_escape}\">"
		end
		if email != null then
			# TODO get more things from github by using the email as a key
			# "https://api.github.com/search/users?q={email}+in:email"
			var md5 = email.md5.to_lower
			res.add "<img src=\"https://secure.gravatar.com/avatar/{md5}?size=20&amp;default=retro\">&nbsp;"
		end
		res.add "{e}"
		if page != null then res.add "</a>"
		res.add "</li>"
	end

	# Recursively generate a level in the file tree of the *content* section
	private fun gen_content_level(ot: OrderedTree[Object], os: Array[Object], res: Template)
	do
		res.add "<ul>\n"
		for o in os do
			res.add "<li>"
			if o isa MGroup then
				var d = ""
				var mdoc = o.mdoc
				if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
				res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
			else if o isa ModulePath then
				var d = ""
				var m = o.mmodule
				if m != null then
					var mdoc = m.mdoc
					if mdoc != null then d = ": {mdoc.html_synopsis.write_to_string}"
				end
				res.add "<strong>{o.name}</strong>{d} ({o.filepath.to_s})"
			else
				abort
			end
			var subs = ot.sub.get_or_null(o)
			if subs != null then gen_content_level(ot, subs, res)
			res.add "</li>\n"
		end
		res.add "</ul>\n"
	end

	# Compute information and generate a full HTML page for a package
	fun package_page(mpackage: MPackage): Writable
	do
		var res = new CatalogPage
		var score = score[mpackage].to_f
		var name = mpackage.name.html_escape
		res.more_head.add """<title>{{{name}}}</title>"""

		res.add """
<div class="content">
<h1 class="package-name">{{{name}}}</h1>
"""
		var mdoc = mpackage.mdoc_or_fallback
		if mdoc != null then
			score += 100.0
			res.add mdoc.html_documentation
			score += mdoc.content.length.score
		end

		res.add "<h2>Content</h2>"
		var ot = new OrderedTree[Object]
		for g in mpackage.mgroups do
			var pa = g.parent
			if g.is_interesting then
				ot.add(pa, g)
				pa = g
			end
			for mp in g.module_paths do
				ot.add(pa, mp)
			end
		end
		ot.sort_with(alpha_comparator)
		gen_content_level(ot, ot.roots, res)


		res.add """
</div>
<div class="sidebar">
<ul class="box">
"""
		var homepage = mpackage.metadata("upstream.homepage")
		if homepage != null then
			score += 5.0
			var e = homepage.html_escape
			res.add "<li><a href=\"{e}\">{e}</a></li>\n"
		end
		var maintainer = mpackage.metadata("package.maintainer")
		if maintainer != null then
			score += 5.0
			add_contrib(maintainer, mpackage, res)
			mpackage.maintainers.add maintainer
			var projs = maint2proj[maintainer]
			if not projs.has(mpackage) then projs.add mpackage
		end
		var license = mpackage.metadata("package.license")
		if license != null then
			score += 5.0
			var e = license.html_escape
			res.add "<li><a href=\"http://opensource.org/licenses/{e}\">{e}</a> license</li>\n"
		end
		res.add "</ul>\n"

		res.add "<h3>Source Code</h3>\n<ul class=\"box\">\n"
		var browse = mpackage.metadata("upstream.browse")
		if browse != null then
			score += 5.0
			var e = browse.html_escape
			res.add "<li><a href=\"{e}\">{e}</a></li>\n"
		end
		var git = mpackage.metadata("upstream.git")
		if git != null then
			var e = git.html_escape
			res.add "<li><tt>{e}</tt></li>\n"
		end
		var last_date = mpackage.last_date
		if last_date != null then
			var e = last_date.html_escape
			res.add "<li>most recent commit: {e}</li>\n"
		end
		var first_date = mpackage.first_date
		if first_date != null then
			var e = first_date.html_escape
			res.add "<li>oldest commit: {e}</li>\n"
		end
		var commits = commits[mpackage]
		if commits != 0 then
			res.add "<li>{commits} commits</li>\n"
		end
		res.add "</ul>\n"

		res.add "<h3>Tags</h3>\n"
		var tags = mpackage.metadata("package.tags")
		var ts2 = new Array[String]
		var cat = null
		if tags != null then
			var ts = tags.split(",")
			for t in ts do
				t = t.trim
				if t == "" then continue
				if cat == null then cat = t
				tag2proj[t].add mpackage
				t = t.html_escape
				ts2.add "<a href=\"index.html#tag_{t}\">{t}</a>"
			end
			res.add_list(ts2, ", ", ", ")
		end
		if ts2.is_empty then
			var t = "none"
			cat = t
			tag2proj[t].add mpackage
			res.add "<a href=\"index.html#tag_{t}\">{t}</a>"
		end
		if cat != null then cat2proj[cat].add mpackage
		score += ts2.length.score

		if deps.has(mpackage) then
			var reqs = deps[mpackage].greaters.to_a
			reqs.remove(mpackage)
			alpha_comparator.sort(reqs)
			res.add "<h3>Requirements</h3>\n"
			if reqs.is_empty then
				res.add "none"
			else
				var list = new Array[String]
				for r in reqs do
					var direct = deps.has_direct_edge(mpackage, r)
					var s = "<a href=\"{r}.html\">"
					if direct then s += "<strong>"
					s += r.to_s
					if direct then s += "</strong>"
					s += "</a>"
					list.add s
				end
				res.add_list(list, ", ", " and ")
			end

			reqs = deps[mpackage].smallers.to_a
			reqs.remove(mpackage)
			alpha_comparator.sort(reqs)
			res.add "<h3>Clients</h3>\n"
			if reqs.is_empty then
				res.add "none"
			else
				var list = new Array[String]
				for r in reqs do
					var direct = deps.has_direct_edge(r, mpackage)
					var s = "<a href=\"{r}.html\">"
					if direct then s += "<strong>"
					s += r.to_s
					if direct then s += "</strong>"
					s += "</a>"
					list.add s
				end
				res.add_list(list, ", ", " and ")
			end

			score += deps[mpackage].greaters.length.score
			score += deps[mpackage].direct_greaters.length.score
			score += deps[mpackage].smallers.length.score
			score += deps[mpackage].direct_smallers.length.score
		end

		var contributors = mpackage.contributors
		if not contributors.is_empty then
			res.add "<h3>Contributors</h3>\n<ul class=\"box\">"
			for c in contributors do
				add_contrib(c, mpackage, res)
			end
			res.add "</ul>"
		end
		score += contributors.length.to_f

		var mmodules = 0
		var mclasses = 0
		var mmethods = 0
		var loc = 0
		for g in mpackage.mgroups do
			mmodules += g.module_paths.length
			for m in g.mmodules do
				var am = modelbuilder.mmodule2node(m)
				if am != null then
					var file = am.location.file
					if file != null then
						loc += file.line_starts.length - 1
					end
				end
				for cd in m.mclassdefs do
					mclasses += 1
					for pd in cd.mpropdefs do
						if not pd isa MMethodDef then continue
						mmethods += 1
					end
				end
			end
		end
		self.mmodules[mpackage] = mmodules
		self.mclasses[mpackage] = mclasses
		self.mmethods[mpackage] = mmethods
		self.loc[mpackage] = loc

		#score += mmodules.score
		score += mclasses.score
		score += mmethods.score
		score += loc.score

		res.add """
<h3>Stats</h3>
<ul class="box">
<li>{{{mmodules}}} modules</li>
<li>{{{mclasses}}} classes</li>
<li>{{{mmethods}}} methods</li>
<li>{{{loc}}} lines of code</li>
</ul>
"""

		res.add """
</div>
"""
		self.score[mpackage] = score.to_i

		return res
	end

	# Return a short HTML sequence for a package
	#
	# Intended to use in lists.
	fun li_package(p: MPackage): String
	do
		var res = ""
		var f = "{p.name}.html"
		res += "<a href=\"{f}\">{p}</a>"
		var d = p.mdoc_or_fallback
		if d != null then res += " - {d.html_synopsis.write_to_string}"
		return res
	end

	# List packages by group.
	#
	# For each key of the `map` a `<h3>` is generated.
	# Each package is then listed.
	#
	# The list of keys is generated first to allow fast access to the correct `<h3>`.
	# `id_prefix` is used to give an id to the `<h3>` element.
	fun list_by(map: MultiHashMap[String, MPackage], id_prefix: String): Template
	do
		var res = new Template
		var keys = map.keys.to_a
		alpha_comparator.sort(keys)
		var list = [for x in keys do "<a href=\"#{id_prefix}{x.html_escape}\">{x.html_escape}</a>"]
		res.add_list(list, ", ", " and ")

		for k in keys do
			var projs = map[k].to_a
			alpha_comparator.sort(projs)
			var e = k.html_escape
			res.add "<h3 id=\"{id_prefix}{e}\">{e} ({projs.length})</h3>\n<ul>\n"
			for p in projs do
				res.add "<li>"
				res.add li_package(p)
				res.add "</li>"
			end
			res.add "</ul>"
		end
		return res
	end

	# List the 10 best packages from `cpt`
	fun list_best(cpt: Counter[MPackage]): Template
	do
		var res = new Template
		res.add "<ul>"
		var best = cpt.sort
		for i in [1..10] do
			if i > best.length then break
			var p = best[best.length-i]
			res.add "<li>"
			res.add li_package(p)
			# res.add " ({cpt[p]})"
			res.add "</li>"
		end
		res.add "</ul>"
		return res
	end

	# Collect more information on a package using the `git` tool.
	fun git_info(mpackage: MPackage)
	do
		var ini = mpackage.ini
		if ini == null then return

		# TODO use real git info
		#var repo = ini.get_or_null("upstream.git")
		#var branch = ini.get_or_null("upstream.git.branch")
		#var directory = ini.get_or_null("upstream.git.directory")

		var dirpath = mpackage.root.filepath
		if dirpath == null then return

		# Collect commits info
		var res = git_run("log", "--no-merges", "--follow", "--pretty=tformat:%ad;%aN <%aE>", "--", dirpath)
		var contributors = new Counter[String]
		var commits = res.split("\n")
		if commits.not_empty and commits.last == "" then commits.pop
		self.commits[mpackage] = commits.length
		for l in commits do
			var s = l.split_once_on(';')
			if s.length != 2 or s.last == "" then continue

			# Collect date of last and first commit
			if mpackage.last_date == null then mpackage.last_date = s.first
			mpackage.first_date = s.first

			# Count contributors
			contributors.inc(s.last)
		end
		for c in contributors.sort.reverse_iterator do
			mpackage.contributors.add c
		end

	end

	# Produce a HTML table containig information on the packages
	#
	# `package_page` must have been called before so that information is computed.
	fun table_packages(mpackages: Array[MPackage]): Template
	do
		alpha_comparator.sort(mpackages)
		var res = new Template
		res.add "<table data-toggle=\"table\" data-sort-name=\"name\" data-sort-order=\"desc\" width=\"100%\">\n"
		res.add "<thead><tr>\n"
		res.add "<th data-field=\"name\" data-sortable=\"true\">name</th>\n"
		res.add "<th data-field=\"maint\" data-sortable=\"true\">maint</th>\n"
		res.add "<th data-field=\"contrib\" data-sortable=\"true\">contrib</th>\n"
		if deps.not_empty then
			res.add "<th data-field=\"reqs\" data-sortable=\"true\">reqs</th>\n"
			res.add "<th data-field=\"dreqs\" data-sortable=\"true\">direct<br>reqs</th>\n"
			res.add "<th data-field=\"cli\" data-sortable=\"true\">clients</th>\n"
			res.add "<th data-field=\"dcli\" data-sortable=\"true\">direct<br>clients</th>\n"
		end
		res.add "<th data-field=\"mod\" data-sortable=\"true\">modules</th>\n"
		res.add "<th data-field=\"cla\" data-sortable=\"true\">classes</th>\n"
		res.add "<th data-field=\"met\" data-sortable=\"true\">methods</th>\n"
		res.add "<th data-field=\"loc\" data-sortable=\"true\">lines</th>\n"
		res.add "<th data-field=\"score\" data-sortable=\"true\">score</th>\n"
		res.add "</tr></thead>"
		for p in mpackages do
			res.add "<tr>"
			res.add "<td><a href=\"{p.name}.html\">{p.name}</a></td>"
			var maint = "?"
			if p.maintainers.not_empty then maint = p.maintainers.first
			res.add "<td>{maint}</td>"
			res.add "<td>{p.contributors.length}</td>"
			if deps.not_empty then
				res.add "<td>{deps[p].greaters.length-1}</td>"
				res.add "<td>{deps[p].direct_greaters.length}</td>"
				res.add "<td>{deps[p].smallers.length-1}</td>"
				res.add "<td>{deps[p].direct_smallers.length}</td>"
			end
			res.add "<td>{mmodules[p]}</td>"
			res.add "<td>{mclasses[p]}</td>"
			res.add "<td>{mmethods[p]}</td>"
			res.add "<td>{loc[p]}</td>"
			res.add "<td>{score[p]}</td>"
			res.add "</tr>\n"
		end
		res.add "</table>\n"
		return res
	end
end

# Execute a git command and return the result
fun git_run(command: String...): String
do
	# print "git {command.join(" ")}"
	var p = new ProcessReader("git", command...)
	var res = p.read_all
	p.close
	p.wait
	return res
end

var model = new Model
var tc = new ToolContext

var opt_dir = new OptionString("Directory where the HTML files are generated", "-d", "--dir")
var opt_no_git = new OptionBool("Do not gather git information from the working directory", "--no-git")
var opt_no_parse = new OptionBool("Do not parse nit files (no importation information)", "--no-parse")
var opt_no_model = new OptionBool("Do not analyse nit files (no class/method information)", "--no-model")

tc.option_context.add_option(opt_dir, opt_no_git, opt_no_parse, opt_no_model)

tc.process_options(sys.args)
tc.keep_going = true

var modelbuilder = new ModelBuilder(model, tc)
var catalog = new Catalog(modelbuilder)

# Get files or groups
for a in tc.option_context.rest do
	modelbuilder.get_mgroup(a)
	modelbuilder.identify_file(a)
end

# Scan packages and compute information
for p in model.mpackages do
	var g = p.root
	assert g != null
	modelbuilder.scan_group(g)

	# Load the module to process importation information
	if opt_no_parse.value then continue
	modelbuilder.parse_group(g)

	catalog.deps.add_node(p)
	for gg in p.mgroups do for m in gg.mmodules do
		for im in m.in_importation.direct_greaters do
			var ip = im.mpackage
			if ip == null or ip == p then continue
			catalog.deps.add_edge(p, ip)
		end
	end
end

if not opt_no_git.value then for p in model.mpackages do
	catalog.git_info(p)
end

# Run phases to modelize classes and properties (so we can count them)
if not opt_no_model.value then
	modelbuilder.run_phases
end

var out = opt_dir.value or else "catalog.out"
out.mkdir

# Generate the css (hard coded)
var css = """
body {
margin-top:			15px;
background-color:		#f8f8f8;
}

a {
color:				#0D8921;
text-decoration:		none;
}

a:hover {
color:				#333;
text-decoration:		none;
}

h1 {
font-weight:			bold;
color:                          #0D8921;
font-size:			22px;
}

h2 {
color:                          #6C6C6C;
font-size:			18px;
border-bottom:			solid 3px #CCC;
}

h3 {
color:                          #6C6C6C;
font-size:			15px;
border-bottom:			solid 1px #CCC;
}

ul {
list-style-type:		square;
}

dd {
color:				#6C6C6C;
margin-top:			1em;
margin-bottom:			1em;
}

pre {
border:				1px solid #CCC;
font-family:			Monospace;
color:				#2d5003;
background-color:		rgb(250, 250, 250);
}

code {
font-family:			Monospace;
color:				#2d5003;
}

footer {
margin-top:			20px;
}

.container {
margin:				0 auto;
padding:			0 20px;
}

.content {
float:				left;
margin-top:			40px;
width:				65%;
}

.sidebar {
float:				right;
margin-top:			40px;
width:				30%
}

.sidebar h3 {
color:                          #0D8921;
font-size:			18px;
border-bottom:			0px;
}

.box {
margin:				0;
padding:			0;
}

.box li {
line-height:			2.5;
white-space:			nowrap;
overflow:			hidden;
text-overflow:			ellipsis;
padding-right:			10px;
border-bottom:			1px solid rgba(0,0,0,0.2);
}
"""
css.write_to_file(out/"style.css")

# PAGES

for p in model.mpackages do
	# print p
	var f = "{p.name}.html"
	catalog.package_page(p).write_to_file(out/f)
end

# INDEX

var index = new CatalogPage
index.more_head.add "<title>Packages in Nit</title>"

index.add """
<div class="content">
<h1>Packages in Nit</h1>
"""

index.add "<h2>Highlighted Packages</h2>\n"
index.add catalog.list_best(catalog.score)

if catalog.deps.not_empty then
	index.add "<h2>Most Required</h2>\n"
	var reqs = new Counter[MPackage]
	for p in model.mpackages do
		reqs[p] = catalog.deps[p].smallers.length - 1
	end
	index.add catalog.list_best(reqs)
end

index.add "<h2>By First Tag</h2>\n"
index.add catalog.list_by(catalog.cat2proj, "cat_")

index.add "<h2>By Any Tag</h2>\n"
index.add catalog.list_by(catalog.tag2proj, "tag_")

index.add """
</div>
<div class="sidebar">
<h3>Stats</h3>
<ul class="box">
<li>{{{model.mpackages.length}}} packages</li>
<li>{{{catalog.maint2proj.length}}} maintainers</li>
<li>{{{catalog.contrib2proj.length}}} contributors</li>
<li>{{{catalog.tag2proj.length}}} tags</li>
<li>{{{catalog.mmodules.sum}}} modules</li>
<li>{{{catalog.mclasses.sum}}} classes</li>
<li>{{{catalog.mmethods.sum}}} methods</li>
<li>{{{catalog.loc.sum}}} lines of code</li>
</ul>
</div>
"""

index.write_to_file(out/"index.html")

# PEOPLE
{'fraction_non_alphanumeric': 0.1477570814504574, 'fraction_numerical': 0.009831367794555275, 'mean_word_length': 3.371783752529633, 'pattern_counts': {'":': 4, '<': 517, '<?xml version=': 0, '>': 511, 'https://': 12, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2532925', 'n_tokens_mistral': 18107, 'n_tokens_neox': 16193, 'n_words': 4665}