<commit_before>link_template="""\
   <ul>
      <li><a href="{lang_code:}/overview/tokenization.html">Tokenization</a></li>
      <li>Morphology
        <ul>
          <li><a href="{lang_code:}/overview/morphology.html">General principles</a></li>
          <li><a href="{lang_code:}/pos/index.html">{lang_name:} POS tags</a> (<a href="{lang_code:}/pos/all.html">single document</a>)</li>
          <li><a href="{lang_code:}/feat/index.html">{lang_name:} features</a> (<a href="{lang_code:}/feat/all.html">single document</a>)</li>
        </ul>
      </li>
      <li>Syntax
        <ul>
          <li><a href="{lang_code:}/overview/syntax.html">General principles</a></li>
          <li><a href="{lang_code:}/overview/specific-syntax.html">Specific constructions</a></li>
          <li><a href="{lang_code:}/dep/index.html">{lang_name:} relations</a> (<a href="{lang_code:}/dep/all.html">single document</a>)</li>
        </ul>
      </li>
    </ul>
"""

import sys
import glob
import os.path
import json
import re
import codecs
import StringIO

no_data_token_count_span="""<span class="widespan" style="color:gray"><span class="hint--top hint--info" data-hint="No corpus data">-</span></span>"""
token_count_span="""<span class="widespan"><span class="hint--top hint--info" data-hint="{token_count:,} tokens {word_count:,} words">{tcountk:,}K</span></span>"""
def get_token_count_span(corpus_data):
    token_count=corpus_data.get("token_count",0)
    if token_count==0: #No data
        return no_data_token_count_span
    else:
        return token_count_span.format(tcountk=token_count//1000,**corpus_data)


def get_column_icons(corpus_data):
    r="""<span class="widespan">"""
    if corpus_data.get("words_with_lemma_count",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Lemmas">&#9409;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    if corpus_data.get("catvals",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Features">&#9403;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    if corpus_data.get("words_with_deps_count",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Secondary dependencies">&#9401;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    r+="""</span>"""
    return r

categories={(u"Documentation status",u"stub"):"""<span class="widespan" style="color:gray"><span class="hint--top hint--info" data-hint="No documentation">-</span></span>""",
            (u"Documentation status",u"partial"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Partial documentation"><i class="fa fa-file-o"></i></span></span>""",
            (u"Documentation status",u"complete"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Complete documentation"><i class="fa fa-file-text-o"></i></span></span>""",
            (u"Data source",u"unknown"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Data source not known">-</span></span>""",
            (u"Data source",u"automatic conversion"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Automatic conversion"><i class="fa fa-cogs"></i></span></span>""",
            (u"Data source",u"automatic conversion + manual check"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Automatic conversion + manual check"><i class="fa fa-cogs"></i><!--<i class="fa fa-plus" style="font-size: 0.75em; line-height: 1.33em; vertical-align: +10%;">--><i class="fa fa-check"></i></span></span>""",
            (u"Data source",u"native UD annotation"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Manual annotation"><i class="fa fa-user"></i></span></span>""",
            (u"License",u"none"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Data not released as of yet">-</span></span>""",
            (u"Data available since",u"UD v1.0"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="First released in UD version 1.0 (Jan 2015)"><i class="fa fa-check"></i></span></span>""",
            (u"Data available since",u"UD v1.1"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="First released in UD version 1.1 (May 2015)"><i class="fa fa-check"></i></span></span>""",
            (u"Data available since",u"UD v1.2"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Will be released in UD version 1.2 (November 2015)"><i class="fa fa-hourglass-end"></i></span></span>""",
            (u"Data available since",u"none"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="No firm schedule for data release.">-</span></span>"""}

license_span="""<span class="widespan"><span class="hint--top hint--info" data-hint="{license:}">{licenseshort:}</span></span>"""
for lic in ("CC BY-NC-SA 3.0","CC BY-NC-SA 3.0 US","CC BY-NC-SA 4.0"):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license"  src="logos/by-nc-sa.svg">""")
for lic in ("GNU GPL Version 2",):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/gpl.svg">""")
for lic in ("CC BY-SA 4.0","CC BY-SA 3.0"):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/by-sa.svg">""")
for lic in ("CC BY 4.0",):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/by.svg">""")

valueRe=re.compile(u"^([a-zA-Z ]+): ([A-Za-z0-9+. -]+)$")
def analyze_readme(dir_name):
    readme_data={u"Documentation status":u"stub",u"Data source":u"unknown",u"License":u"none",u"Data available since":u"none"}
    readmes=sorted(x for x in glob.glob(os.path.join(dir_name,"*")) if "readme" in x.lower())
    if not readmes: #No readme file!
        return readme_data
    with codecs.open(readmes[0],"r","utf-8") as f:
        for line in f:
            match=valueRe.match(line)<commit_msg>Fixing some of the tip texts<commit_after>link_template="""\
   <ul>
      <li><a href="{lang_code:}/overview/tokenization.html">Tokenization</a></li>
      <li>Morphology
        <ul>
          <li><a href="{lang_code:}/overview/morphology.html">General principles</a></li>
          <li><a href="{lang_code:}/pos/index.html">{lang_name:} POS tags</a> (<a href="{lang_code:}/pos/all.html">single document</a>)</li>
          <li><a href="{lang_code:}/feat/index.html">{lang_name:} features</a> (<a href="{lang_code:}/feat/all.html">single document</a>)</li>
        </ul>
      </li>
      <li>Syntax
        <ul>
          <li><a href="{lang_code:}/overview/syntax.html">General principles</a></li>
          <li><a href="{lang_code:}/overview/specific-syntax.html">Specific constructions</a></li>
          <li><a href="{lang_code:}/dep/index.html">{lang_name:} relations</a> (<a href="{lang_code:}/dep/all.html">single document</a>)</li>
        </ul>
      </li>
    </ul>
"""

import sys
import glob
import os.path
import json
import re
import codecs
import StringIO

no_data_token_count_span="""<span class="widespan" style="color:gray"><span class="hint--top hint--info" data-hint="No corpus data">-</span></span>"""
token_count_span="""<span class="widespan"><span class="hint--top hint--info" data-hint="{token_count:,} tokens {word_count:,} words {tree_count:,} sentences">{tcountk:,}K</span></span>"""
def get_token_count_span(corpus_data):
    token_count=corpus_data.get("token_count",0)
    if token_count==0: #No data
        return no_data_token_count_span
    else:
        return token_count_span.format(tcountk=token_count//1000,**corpus_data)


def get_column_icons(corpus_data):
    r="""<span class="widespan">"""
    if corpus_data.get("words_with_lemma_count",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Lemmas">&#9409;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    if corpus_data.get("catvals",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Features">&#9403;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    if corpus_data.get("words_with_deps_count",0)>0:
        r+="""<span class="tagspan"><span class="hint--top hint--info" data-hint="Secondary dependencies">&#9401;</span></span>"""
    else:
        r+="""<span class="tagspan"></span>"""
    r+="""</span>"""
    return r

categories={(u"Documentation status",u"stub"):"""<span class="widespan" style="color:gray"><span class="hint--top hint--info" data-hint="No documentation">-</span></span>""",
            (u"Documentation status",u"partial"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Partial documentation"><i class="fa fa-file-o"></i></span></span>""",
            (u"Documentation status",u"complete"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Complete documentation"><i class="fa fa-file-text-o"></i></span></span>""",
            (u"Data source",u"unknown"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Data source not known">-</span></span>""",
            (u"Data source",u"automatic conversion"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Automatic conversion"><i class="fa fa-cogs"></i></span></span>""",
            (u"Data source",u"automatic conversion + manual check"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Automatic conversion with manual corrections"><i class="fa fa-cogs"></i><!--<i class="fa fa-plus" style="font-size: 0.75em; line-height: 1.33em; vertical-align: +10%;">--><i class="fa fa-check"></i></span></span>""",
            (u"Data source",u"native UD annotation"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Manual annotation"><i class="fa fa-user"></i></span></span>""",
            (u"License",u"none"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="License not known">-</span></span>""",
            (u"Data available since",u"UD v1.0"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="First released in UD version 1.0 (Jan 2015)"><i class="fa fa-check"></i></span></span>""",
            (u"Data available since",u"UD v1.1"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="First released in UD version 1.1 (May 2015)"><i class="fa fa-check"></i></span></span>""",
            (u"Data available since",u"UD v1.2"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="Will be released in UD version 1.2 (November 2015)"><i class="fa fa-hourglass-end"></i></span></span>""",
            (u"Data available since",u"none"):"""<span class="widespan"><span class="hint--top hint--info" data-hint="No firm schedule for data release">-</span></span>"""}

license_span="""<span class="widespan"><span class="hint--top hint--info" data-hint="{license:}">{licenseshort:}</span></span>"""
for lic in ("CC BY-NC-SA 3.0","CC BY-NC-SA 3.0 US","CC BY-NC-SA 4.0"):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license"  src="logos/by-nc-sa.svg">""")
for lic in ("GNU GPL Version 2",):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/gpl.svg">""")
for lic in ("CC BY-SA 4.0","CC BY-SA 3.0"):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/by-sa.svg">""")
for lic in ("CC BY 4.0",):
    categories[(u"License",lic)]=license_span.format(license=lic,licenseshort="""<img class="license" src="logos/by.svg">""")

valueRe=re.compile(u"^([a-zA-Z ]+): ([A-Za-z0-9+. -]+)$")
def analyze_readme(dir_name):
    readme_data={u"Documentation status":u"stub",u"Data source":u"unknown",u"License":u"none",u"Data available since":u"none"}
    readmes=sorted(x for x in glob.glob(os.path.join(dir_name,"*")) if "readme" in x.lower())
    if not readmes: #No readme file!
        return readme_data
    with codecs.open(readmes[0],"r","utf-8") as f:
        for line in f:
            match=valueRe.match(line)
{'fraction_non_alphanumeric': 0.23451183911774245, 'fraction_numerical': 0.011839117742458645, 'mean_word_length': 4.678176795580111, 'pattern_counts': {'":': 8, '<': 299, '<?xml version=': 0, '>': 305, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4975482', 'n_tokens_mistral': 4349, 'n_tokens_neox': 4221, 'n_words': 722}