<commit_before>#!/usr/bin/python
# -*- coding: utf-8  -*-
'''
Mechanism for identifying UGC-objects in lists and adding these to the database
Known problems:
* This is sensitive to the structure of the page. 
  e.g. one parameter per row table finished on a separate row etc.
  Problem with using the available common.findUnit is that this 
  makes it hard to identify where on the page to add the id.
  Alternative would be to parse each object using common.findUnit
  and re-output it. This has the benifit of structuring unstructured pages
  but risks messing up pages with simple typos.
'''

def run(checkList, wpApi, dbWriteSQL):
    changed=0
    log=''
    checkPages = wpApi.getPage(checkList)
    for pagename, contents  in checkPages.iteritems():
        changes = UGConPage(pagename, contents, dbWriteSQL)
        if changes:
            changed = changed + changes[0]
            wpApi.editText(pagename, changes[1], u'Lägger till UGC-id', minor=True, bot=True, userassert='bot')
            log = u'%s\nUGC objects added to %s' %(log,pagename)
    return (log, changed)<commit_msg>Tweak to log<commit_after>#!/usr/bin/python
# -*- coding: utf-8  -*-
'''
Mechanism for identifying UGC-objects in lists and adding these to the database
Known problems:
* This is sensitive to the structure of the page. 
  e.g. one parameter per row table finished on a separate row etc.
  Problem with using the available common.findUnit is that this 
  makes it hard to identify where on the page to add the id.
  Alternative would be to parse each object using common.findUnit
  and re-output it. This has the benifit of structuring unstructured pages
  but risks messing up pages with simple typos.
'''

def run(checkList, wpApi, dbWriteSQL):
    changed=0
    log=''
    checkPages = wpApi.getPage(checkList)
    for pagename, contents  in checkPages.iteritems():
        changes = UGConPage(pagename, contents, dbWriteSQL)
        if changes:
            changed = changed + changes[0]
            wpApi.editText(pagename, changes[1], u'Lägger till UGC-id', minor=True, bot=True, userassert='bot')
            log = u'%sUGC objects added to %s\n' %(log,pagename)
    return (log, changed)
{'fraction_non_alphanumeric': 0.08687700045724737, 'fraction_numerical': 0.003657978966620942, 'mean_word_length': 3.3326732673267325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4356631', 'n_tokens_mistral': 652, 'n_tokens_neox': 640, 'n_words': 257}