<jupyter_start><jupyter_text>previous: [start](start.ipynb)--- Sharing data features<jupyter_code>import collections
import os

from tf.app import use
A = use('dss', hoist=globals(), silent='deep')
silentOff()<jupyter_output><empty_output><jupyter_text>Making dataA certain consonant gets `cert=100`.Penalties:feature | value | penalty | description--- | --- | --- | ---unc | 1 | 10 | uncertainty level 1unc | 2 | 20 | uncertainty level 2unc | 3 | 30 | uncertainty level 3unc | 4 | 40 | uncertainty level 4rec | 1 | 45 | modern reconstructionrem | 1 | 40 | modern removalrem | 2 | 20 | ancient removalcor | 1 | 18 | modern correctioncor | 2 | 12 | ancient correctionalt | 1 | 25 | alternate readingThe minimum is `1`.We extend the `cert` measure to words, fragments and scrolls by averaging.  The measure function<jupyter_code>def measure(s):
  c = 100
  d = F.unc.v(s)
  if d:
    c -= 10 * d
  d = F.rec.v(s)
  if d:
    c -= 45
  d = F.rem.v(s)
  if d == 1:
    c -= 40
  elif d == 2:
    c -= 20
  d = F.cor.v(s)
  if d == 2 or d == 3:
    c -= 12
  elif d == 1:
    c -= 18
  d = F.alt.v(s)
  if d:
    c -= 25
  if c < 1:
    c = 1
  return c<jupyter_output><empty_output><jupyter_text>Measure everything<jupyter_code>CONS = 'cons'
cert = {}

indent(reset=True)

for sc in F.otype.s('scroll'):
  fN = 0
  fSum = 0
  
  for f in L.d(sc, otype='fragment'):
    lN = 0
    lSum = 0
    
    for l in L.d(f, otype='line'):
      wN = 0
      wSum = 0
      
      for w in L.d(l, otype='word'):
        sN = 0
        sSum = 0
        
        for s in L.d(w, otype='sign'):
          if F.type.v(s) != CONS:
            continue
          sCert = measure(s)
          cert[s] = sCert
          sN += 1
          sSum += sCert
          
        if sN:
          wCert = int(round(sSum / sN))
          cert[w] = wCert
          wN += 1
          wSum += wCert
          
      if wN:
        lCert = int(round(wSum / wN))
        cert[l] = lCert
        lN += 1
        lSum += lCert
        
    if lN:
      fCert = int(round(lSum / lN))
      cert[f] = fCert
      fN += 1
      fSum += fCert
      
  if fN:
    scCert = int(round(fSum / fN))
    cert[sc] = scCert
    
info(f'{len(cert)} certainties determined')<jupyter_output>5.42s 1625373 certainties determined<jupyter_text>Saving data  Location<jupyter_code>GITHUB = os.path.expanduser('~/github')
ORG = 'ancient-data'
REPO = 'tftutorials'
PATH = 'dss/cert'
VERSION = A.version<jupyter_output><empty_output><jupyter_text>Metadata<jupyter_code>metaData = {
  'cert': dict(
    valueType='int',
    description='measure of certainty of material, between 1 and 100 (most certain)',
    creator='<NAME>',
  ),
}<jupyter_output><empty_output><jupyter_text>Save action<jupyter_code>TF.save(nodeFeatures=dict(cert=cert), metaData=metaData, location=f'{GITHUB}/{ORG}/{REPO}/{PATH}/tf', module=VERSION)<jupyter_output>0.00s Exporting 1 node and 0 edge and 0 config features to /Users/dirk/github/ancient-data/tftutorials/dss/cert/tf/0.6:
   |     1.90s T cert                 to /Users/dirk/github/ancient-data/tftutorials/dss/cert/tf/0.6
  1.90s Exported 1 node features and 0 edge features and 0 config features to /Users/dirk/github/ancient-data/tftutorials/dss/cert/tf/0.6<jupyter_text>Share the data<jupyter_code>%%bash -s "$GITHUB" "$ORG" "$REPO" 
cd $1/$2/$3
git add --all .
git commit -m "cert feature added"
git push origin master<jupyter_output>[master e940320] cert feature added
 14 files changed, 6832 insertions(+), 1001 deletions(-)
 create mode 100644 HLL_poster.pdf
 rewrite oldbabylonian/pos.ipynb (87%)
{'fraction_non_alphanumeric': 0.1362237762237762, 'fraction_numerical': 0.03804195804195804, 'mean_word_length': 2.6341463414634148, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '358642', 'license': 'MIT', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1327, 'n_words': 347, 'path': 'dss/share.ipynb', 'repo_name': 'ancient-data/tftutorials', 'size': '3580'}