<commit_before>from __future__ import print_function

import sys
from re import sub

from pyspark.sql import SparkSession, Row
from pyspark.sql.functions import col, udf, regexp_replace

def formatPassage(r):
    text = ""
    if r.url:
        text += "<h2><a href=\"%s\">%s</a></h2>" % (r.url, r.title)
    else:
        text += "<h2>%s</h2>" % r.title
    cluster = "cl" + str(r.cluster)
    dateline = '<date tokenizetagcontent="false">%s</date>' % r.date
    if r.placeOfPublication:
        dateline += ' &middot; <place>%s</place>' % r.placeOfPublication
    text += "<h4>%s</h4>" % dateline
    text += sub('(?<!\\\\)\\n', '<br/>\\n', r.text)
    text += ' <archiveid tokenizetagcontent="false">%s</archiveid>' % cluster
    text += ' <series tokenizetagcontent="false">%s</series>' % r.series
    text += ' <id tokenizetagcontent="false">%s</id>' % r.id
    return Row(archiveid=cluster, id=r.id, imagecount=r.size, title=r.title, date=r.date, placeOfPublication=r.placeOfPublication,
               text=text, page_access=r.page_access)
    

def formatPassages(x):
    (cluster, riter) = x
    rows = list(riter)
    rows.sort(key=lambda z: z.date)
    res = list()
    for i in range(len(rows)):
        r = rows[i].asDict()
        name = "%s_%d" % (cluster, i)
        r['name'] = name
        r['seq'] = i
        r['identifier'] = cluster
        r['pageNumber'] = i
        res.append(Row(**r))
    return res

def formatCluster(x):
    (cluster, riter) = x
    rows = list(riter)<commit_msg>Display id if title missing.<commit_after>from __future__ import print_function

import sys
from re import sub

from pyspark.sql import SparkSession, Row
from pyspark.sql.functions import col, udf, regexp_replace

def formatPassage(r):
    text = ""
    if r.url:
        text += "<h2><a href=\"%s\">%s</a></h2>" % (r.url, (r.title or r.id))
    else:
        text += "<h2>%s</h2>" % (r.title or r.id)
    cluster = "cl" + str(r.cluster)
    dateline = '<date tokenizetagcontent="false">%s</date>' % r.date
    if r.placeOfPublication:
        dateline += ' &middot; <place>%s</place>' % r.placeOfPublication
    text += "<h4>%s</h4>" % dateline
    text += sub('(?<!\\\\)\\n', '<br/>\\n', r.text)
    text += ' <archiveid tokenizetagcontent="false">%s</archiveid>' % cluster
    text += ' <series tokenizetagcontent="false">%s</series>' % r.series
    text += ' <id tokenizetagcontent="false">%s</id>' % r.id
    if r.subject:
        text += ' <subject>%s</subject>' % r.subject
    return Row(archiveid=cluster, id=r.id, imagecount=r.size, title=r.title, date=r.date, placeOfPublication=r.placeOfPublication,
               text=text, page_access=r.page_access)
    

def formatPassages(x):
    (cluster, riter) = x
    rows = list(riter)
    rows.sort(key=lambda z: z.date)
    res = list()
    for i in range(len(rows)):
        r = rows[i].asDict()
        name = "%s_%d" % (cluster, i)
        r['name'] = name
        r['seq'] = i
        r['identifier'] = cluster
        r['pageNumber'] = i
        res.append(Row(**r))
    return res

def formatCluster(x):
    (cluster, riter) = x
    rows = list(riter)
{'fraction_non_alphanumeric': 0.18277848911651728, 'fraction_numerical': 0.0038412291933418692, 'mean_word_length': 2.920953575909661, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008762', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1122, 'n_words': 244}