<commit_before># -*- coding: utf-8  -*-

"""
Main configuration
"""

"""
    * FILE_DIR        : location for input and output files (ended with slash /)
    * INDEX_URL       : list of URLs for source index pages of news
    * NEWS_NUM        : number of news to capture at one time
    * ID_PATTERN      : pattern for matching news ID
    * Q_PATTERN       : quasi-pattern URL string for substituting news ID
    * API_URL         : URL of MediaWiki API (URI of path to api.php)
    * INVALID_PATTERN : nullify news that match any patterns listed
    * SUB_FILTER      : replacement patterns
    * EXCEPTION_LIST  : exceptions for semantic parse matching which generates wiki links
    * TEMPLATES       : dictionary mapping target template page at MediaWiki sites

"""
FILE_DIR     = "/home/anshichao/newscapture/local/"
INDEX_URL    = [
                "http://news.ustb.edu.cn/html/Article_Class2_1_1.html", 
                "http://news.ustb.edu.cn/html/Article_Class2_1_2.html",
               ]

NEWS_NUM    = 4
ID_PATTERN  = r"Article_Show\.asp\?ArticleID=(\d+)"
Q_PATTERN   = "http://news.ustb.edu.cn/html/article/Article_show%s.html"
API_URL     = "http://wiki.ibeike.com/api.php" 
INVALID_PATTERN = [
                   u"^.+：",        # Derivative news prefix (e.g. 光明日报：)
                   u"【视频】",      # Those with [Video] tags
                   r"\S+\s+\S+",    # Those with whitespace(s) in the middle
                   r"\S+&nbsp;\S+", # Those with HTML whitespace in the middle
                   ] 

SUB_FILTER = {
              u".+——(.+)" : r"\1",   # Preserve text after "--"
              u"我校"       : u"北京科技大学",
              u"我国"       : u"中国",
              }

EXCEPTION_LIST = [
                  u"北京科技大学", 
                  u"北京科技大"
                  ]

TEMPLATES = {
                 'news_ref'  :  u"Template:NewsAuto",
                 'news_unref':  u"Template:NewsAutoUntagged",
                 }<commit_msg>fix sub_filter unicode strings<commit_after># -*- coding: utf-8  -*-

"""
Main configuration
"""

"""
    * FILE_DIR        : location for input and output files (ended with slash /)
    * INDEX_URL       : list of URLs for source index pages of news
    * NEWS_NUM        : number of news to capture at one time
    * ID_PATTERN      : pattern for matching news ID
    * Q_PATTERN       : quasi-pattern URL string for substituting news ID
    * API_URL         : URL of MediaWiki API (URI of path to api.php)
    * INVALID_PATTERN : nullify news that match any patterns listed
    * SUB_FILTER      : replacement patterns
    * EXCEPTION_LIST  : exceptions for semantic parse matching which generates wiki links
    * TEMPLATES       : dictionary mapping target template page at MediaWiki sites

"""
FILE_DIR     = "/home/anshichao/newscapture/local/"
INDEX_URL    = [
                "http://news.ustb.edu.cn/html/Article_Class2_1_1.html", 
                "http://news.ustb.edu.cn/html/Article_Class2_1_2.html",
               ]

NEWS_NUM    = 4
ID_PATTERN  = r"Article_Show\.asp\?ArticleID=(\d+)"
Q_PATTERN   = "http://news.ustb.edu.cn/html/article/Article_show%s.html"
API_URL     = "http://wiki.ibeike.com/api.php" 
INVALID_PATTERN = [
                   u"^.+：",        # Derivative news prefix (e.g. 光明日报：)
                   u"【视频】",      # Those with [Video] tags
                   r"\S+\s+\S+",    # Those with whitespace(s) in the middle
                   r"\S+&nbsp;\S+", # Those with HTML whitespace in the middle
                   ] 

SUB_FILTER = {
              u"\S+\u2014\u2014(\S+)" : r"\1", # \u2014 and \u2015 are common Chinese dashes 
              u"\S+\u2015\u2015(\S+)" : r"\1",
              u"我校"       : u"北京科技大学",
              u"我国"       : u"中国",
              }

EXCEPTION_LIST = [
                  u"北京科技大学", 
                  u"北京科技大",
                  u"勤",
                  u"党委",
                  ]

TEMPLATES = {
                 'news_ref'  :  u"Template:NewsAuto",
                 'news_unref':  u"Template:NewsAutoUntagged",
                 }
{'fraction_non_alphanumeric': 0.11814137419673752, 'fraction_numerical': 0.010627780523974296, 'mean_word_length': 1.6908244680851063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898288', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1313, 'n_words': 303}