<commit_before># -*- coding: utf-8 -*-
"""\
This module tries to detect acronyms and cables w/o subject.
"""
import os
import re
import codecs
from functools import partial
from collections import defaultdict
from cablemap.core import cables_from_directory
from cablemap.core.reader import parse_subject

import logging 
import sys
logger = logging.getLogger('cablemap.core.reader')
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))

_AC_PATTERN = re.compile(r'\(([A-Z\-/\.]{2,}[0-9]*)\)')

_KNOWN_TAGS = ('<NAME>', "RICE, CONDOLEEZZA", "CLINTON, HILLARY", "<NAME>", "BUSH, GEORGE",
               "NEW ZEALAND", "<NAME>", "ECONOMY AND FINANCE", "ECONOMIC AFFAIRS", "HUMAN RIGHTS",
               "ITALY NATIONAL ELECTIONS", "ZOELLICK, ROBERT", "ZANU-PF", "GAZA DISENGAGEMENT",
               "COUNTERTERRORISM", "IRAQI FREEDOM", "MEETINGS WITH AMBASSADOR", "EXTERNAL", "CONSULAR AFFAIRS",
               "<NAME>", "ITALIAN POLITICS", "ISRAELI PALESTINIAN AFFAIRS", "DOMESTIC POLITICS",
               "CROS, GERARD", "GLOBAL DEFENSE", "ISRAEL RELATIONS", "MILITARY RELATIONS", "POLITICS FOREIGN POLICY",
               "USEU BRUSSELS", "INDIA-BURMA", "<NAME>.", "<NAME>", "<NAME>",
               "COUNTRY CLEARANCE", "CARICOM", "INMARSAT", "UNESCO", "INTERPOL", "INTELSAT", u'CYPRUS',
               u'MEDIA REACTION', u'UNICEF', u'SPECIALIST', u'POLITICAL PARTIES', u'OFFICIALS', u'UNFICYP',
               u'ECOSOC', u'MERCOSUR', u'AFGHANISTAN', u'MACEDONIA', u'MQADHAFI', u'GOI EXTERNAL', u'GOI INTERNAL',
               u'INTERNAL', u'EXTERNAL', u'ISLAMISTS', u'UNCTAD', u'MINUSTAH', u'UNIDROIT', u'CAMBODIA',
               u'COPUOS', u'UNPUOS', u'ATPDEA', u'AMCHAMS', u'POLITICAL', u'POLINT', u'REGION',
               u'FOREIGN POL', u'MEDIA REACTION REPORT', u'TIP IN TURKEY', u'TURKEY', u'HEBRON', u'KUWAIT-IRAQ RELATIONS',
               u'TERRORISM', u'AGRICULTURE', u'ELECTIONS', u'PHALANAGE PARTY', u'ANACHISTS', u'ECONOMIC', u'ECONOMICS',
               u'ANARCHISTS',
               )

_ACRONYMS = (u'AA/S', u'ADC', u'AFM', u'AG', u'ASD/ISA', u'AU', u'AK', u'APHSCT',
             u'AF-PAK', u'AKP', u'ASD', u'AQAP', u'AQIM', u'ARENA',
             u'BBC', u'BP', u'BR-3',
             u'CMC', u'CNP', u'CODEL', u'CJCS', u'CT', u'CWS/BWC', u'CW', u'CENTCOM',
             u'CDR', u'CFE', u'CISMOA', u'CN', u'CIA', u'CTJWG', u'CG', u'CL', u'CBD',
             u'DAS', u'DCA', u'DDR', u'DEA', u'DG', u'DCM', u'DFID',
             u'DRC', u'DASD', u'DIO', u'DHS', u'DOL', u'DPRK',
             u'EFCC', u'ETA', u'EU', u'EU/US', u'EXBS', u'EUR', u'EFTA',
             u'FATF', u'FBI', u'FCO', u'FDA', u'FDP', u'FM', u'FTAA', u'FARC', u'FX2', u'FMLN',
             u'GAERC', u'GCC', u'GDRC', u'GM', u'GOAJ', u'GOB', u'GOC', u'GOE', u'GOI', u'GOK',
             u'GOL', u'GPC', u'GSL', u'GSP', u'GTMO', u'GOF', u'GOS', u'GBRV', u'GOP', u'GOU',
             u'GFA', u'G/TIP', u'GI',
             u'HMG', u'HLDG', u'HLG', u'HEU',
             u'ICTY', u'II', u'III', u'IMF', u'ITGA', u'IPR', u'IRGC', u'ICAO', u'IRA', u'ISAF',
             u'(INCSR)', u'ICRC', u'ISA', u'INR/B', u'ICC', u'ICG-G', u'ICJ', u'ILSA',
             u'JSF', u'JHA', u'JCET', u'JPMG',
             u'MDC', u'MEP', u'MFA', u'MOD', u'MI-17', u'MI-6',
             u'MRE', u'MP', u'MONUC', u'MOP-3', u'MEA',
             u'NATO', u'NDP', u'NSA', u'NGO', u'NEA', u'NEA/MAG', u'NTM-I',
             u'OIC', u'OECD', u'OAS', u'OSCE', u'OPEC',
             u'PA', u'PD', u'PM', u'PMDB', u'PS', u'PDAS', u'PRT', u'POC', u'PRC/DPRK',
             u'PNG', u'PRC',
             u'ROK', u'RWE', u'RFG', u'RMB', u'RSO', u'RPO', u'RTG',
             u'SLA', u'SLA/U', u'SPD', u'SWIFT', u'S/WCI', u'S/CT', u'S/CRS',
             u'S/GC', u'SCSL', u'S/SRAP', u'SG', u'SAS',
             u'TFTP', u'TFTP/SWIFT', u'U.S.-EU', u'U.S.-UK', u'UAE', u'UK', u'UN',
             u'UNHCR', u'UNSC', u'US', u'US-CU', u'US-EU', u'USG', u'USTR', u'UNCHR',
             u'USEB', u'UNGA', u'U.S./UK', u'UNESCO', u'U/SYG', u'US-ROYG', u'UNSCR',
             u'USG-GOB', u'USD/P', u'USS', u'UNDP',
             u'VFM', u'VP', u'VI', u'VARIG', u'VOA',
             u'WEF', u'WTO',
             u'XVI',
             u'ZANU-PF')

_UNWANTED = (u'SAVE', u'CITES', u'SHARIA', u'IRAN', u'WHO',
             u'CAN', u'SAO', u'IT', u'POSITION')

def run_update(in_dir, predicate=None):
    acronyms = set(_ACRONYMS)
    subjects = set()
    tags = defaultdict(list)
    for cable in cables_from_directory(in_dir, predicate):
        update_acronyms(cable, acronyms)
        update_missing_subjects(cable, subjects)
        update_tags(cable, tags)
    return {'acronyms': acronyms, 'subjects': subjects, 'tags': tags}

def update_acronyms(cable, acronyms):
    if not cable.subject:
        return
    subject_words = [w for w in cable.subject.upper().split() if not w.startswith('XXXXX')]
    acronyms.update((ac for ac in _AC_PATTERN.findall(cable.content) if ac in subject_words and ac not in _UNWANTED))

def update_missing_subjects(cable, cable_refs):
    if not parse_subject(cable.content, fix_subject=False):
        cable_refs.add(cable.reference_id)

def update_tags(cable, tags):
    for tag in cable.tags:
        if len(tag) > 5 and tag not in _KNOWN_TAGS:
            tags[tag].append(cable.reference_id)
 
def _filename(name):
    return os.path.join(os.path.dirname(__file__), name)

def _file_in_core(name):
    return os.path.join(os.path.dirname(__file__), '..', 'cablemap.core', 'cablemap/core/', name)

def _file_as_set(filename):
    if not os.path.exists(filename):
        return set()
    f = codecs.open(filename, 'rb', 'utf-8')
    s = set((l.rstrip() for l in f))
    f.close()
    return s

def _write_set(filename, s):
    f = codecs.open(filename, 'wb', 'utf-8')
    for item in sorted(s):
        f.write(item)
        f.write('\n')
    f.close()

def update_file(filename, found):
    if not found:
        return
    current = _file_as_set(filename)
    diff = current ^ found
    if diff:
        _write_set(filename, found)
        removed = current - found
        fn = os.path.basename(filename)
        header = 'Changes in %s' % fn
        print('\n' + header)
        print('=' * len(header))
        if removed:
            print('Removed:\n%r\n' % (removed))
        added = found - current
        if added:
            print('Added:\n%r\n' % (added))


_FILE_ACRONYMS = _file_in_core('acronyms.txt')
_FILE_SUBJECTS = _filename('no_subject.txt')
    
if __name__ == '__main__':
    if not os.path.isdir('./cable/'):
        raise Exception('Expected a directory "cable"')
    import pprint
    pp = pprint.PrettyPrinter(indent=4)
    res = run_update('./cable/')
    update_file(_FILE_ACRONYMS, res['acronyms'])
    update_file(_FILE_SUBJECTS, res['subjects'])
    print('Valid TAGs?')
    pp.pprint(res['tags'])<commit_msg>Added known TAGs, fixed printing of dct<commit_after># -*- coding: utf-8 -*-
"""\
This module tries to detect acronyms and cables w/o subject.
"""
import os
import re
import codecs
from functools import partial
from collections import defaultdict
from cablemap.core import cables_from_directory
from cablemap.core.reader import parse_subject

import logging 
import sys
logger = logging.getLogger('cablemap.core.reader')
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))

_AC_PATTERN = re.compile(r'\(([A-Z\-/\.]{2,}[0-9]*)\)')

_KNOWN_TAGS = ('<NAME>', "RICE, CONDOLEEZZA", "CLINTON, HILLARY", "<NAME>", "BUSH, GEORGE",
               "NEW ZEALAND", "ROOD, JOHN", "ECONOMY AND FINANCE", "ECONOMIC AFFAIRS", "HUMAN RIGHTS",
               "ITALY NATIONAL ELECTIONS", "ZOELLICK, ROBERT", "ZANU-PF", "GAZA DISENGAGEMENT",
               "COUNTERTERRORISM", "IRAQI FREEDOM", "MEETINGS WITH AMBASSADOR", "EXTERNAL", "CONSULAR AFFAIRS",
               "JIMENEZ, GASPAR", "ITALIAN POLITICS", "ISRAELI PALESTINIAN AFFAIRS", "DOMESTIC POLITICS",
               "CROS, GERARD", "GLOBAL DEFENSE", "ISRAEL RELATIONS", "MILITARY RELATIONS", "POLITICS FOREIGN POLICY",
               "USEU BRUSSELS", "INDIA-BURMA", "<NAME>.", "<NAME>", "<NAME>",
               "COUNTRY CLEARANCE", "CARICOM", "INMARSAT", "UNESCO", "INTERPOL", "INTELSAT", u'CYPRUS',
               u'MEDIA REACTION', u'UNICEF', u'SPECIALIST', u'POLITICAL PARTIES', u'OFFICIALS', u'UNFICYP',
               u'ECOSOC', u'MERCOSUR', u'AFGHANISTAN', u'MACEDONIA', u'MQADHAFI', u'GOI EXTERNAL', u'GOI INTERNAL',
               u'INTERNAL', u'EXTERNAL', u'ISLAMISTS', u'UNCTAD', u'MINUSTAH', u'UNIDROIT', u'CAMBODIA',
               u'COPUOS', u'UNPUOS', u'ATPDEA', u'AMCHAMS', u'POLITICAL', u'POLINT', u'REGION',
               u'FOREIGN POL', u'MEDIA REACTION REPORT', u'TIP IN TURKEY', u'TURKEY', u'HEBRON', u'KUWAIT-IRAQ RELATIONS',
               u'TERRORISM', u'AGRICULTURE', u'ELECTIONS', u'PHALANAGE PARTY', u'ANACHISTS', u'ECONOMIC', u'ECONOMICS',
               u'ANARCHISTS', u'<NAME>.', u'<NAME>', '<NAME>', u'<NAME>', u'<NAME>',
               u'<NAME>', u'<NAME>', u'SECRETARY OF COMMERCE', u'BURNS, WILLIAM',
               u'ISRAELI SOCIETY', u'ENVIRONMENT SCIENCE AND TECHNOLOGY',
               )

_ACRONYMS = (u'AA/S', u'ADC', u'AFM', u'AG', u'ASD/ISA', u'AU', u'AK', u'APHSCT',
             u'AF-PAK', u'AKP', u'ASD', u'AQAP', u'AQIM', u'ARENA',
             u'BBC', u'BP', u'BR-3',
             u'CMC', u'CNP', u'CODEL', u'CJCS', u'CT', u'CWS/BWC', u'CW', u'CENTCOM',
             u'CDR', u'CFE', u'CISMOA', u'CN', u'CIA', u'CTJWG', u'CG', u'CL', u'CBD',
             u'DAS', u'DCA', u'DDR', u'DEA', u'DG', u'DCM', u'DFID',
             u'DRC', u'DASD', u'DIO', u'DHS', u'DOL', u'DPRK',
             u'EFCC', u'ETA', u'EU', u'EU/US', u'EXBS', u'EUR', u'EFTA',
             u'FATF', u'FBI', u'FCO', u'FDA', u'FDP', u'FM', u'FTAA', u'FARC', u'FX2', u'FMLN',
             u'GAERC', u'GCC', u'GDRC', u'GM', u'GOAJ', u'GOB', u'GOC', u'GOE', u'GOI', u'GOK',
             u'GOL', u'GPC', u'GSL', u'GSP', u'GTMO', u'GOF', u'GOS', u'GBRV', u'GOP', u'GOU',
             u'GFA', u'G/TIP', u'GI',
             u'HMG', u'HLDG', u'HLG', u'HEU',
             u'ICTY', u'II', u'III', u'IMF', u'ITGA', u'IPR', u'IRGC', u'ICAO', u'IRA', u'ISAF',
             u'(INCSR)', u'ICRC', u'ISA', u'INR/B', u'ICC', u'ICG-G', u'ICJ', u'ILSA',
             u'JSF', u'JHA', u'JCET', u'JPMG',
             u'MDC', u'MEP', u'MFA', u'MOD', u'MI-17', u'MI-6',
             u'MRE', u'MP', u'MONUC', u'MOP-3', u'MEA',
             u'NATO', u'NDP', u'NSA', u'NGO', u'NEA', u'NEA/MAG', u'NTM-I',
             u'OIC', u'OECD', u'OAS', u'OSCE', u'OPEC',
             u'PA', u'PD', u'PM', u'PMDB', u'PS', u'PDAS', u'PRT', u'POC', u'PRC/DPRK',
             u'PNG', u'PRC',
             u'ROK', u'RWE', u'RFG', u'RMB', u'RSO', u'RPO', u'RTG',
             u'SLA', u'SLA/U', u'SPD', u'SWIFT', u'S/WCI', u'S/CT', u'S/CRS',
             u'S/GC', u'SCSL', u'S/SRAP', u'SG', u'SAS',
             u'TFTP', u'TFTP/SWIFT', u'U.S.-EU', u'U.S.-UK', u'UAE', u'UK', u'UN',
             u'UNHCR', u'UNSC', u'US', u'US-CU', u'US-EU', u'USG', u'USTR', u'UNCHR',
             u'USEB', u'UNGA', u'U.S./UK', u'UNESCO', u'U/SYG', u'US-ROYG', u'UNSCR',
             u'USG-GOB', u'USD/P', u'USS', u'UNDP',
             u'VFM', u'VP', u'VI', u'VARIG', u'VOA',
             u'WEF', u'WTO',
             u'XVI',
             u'ZANU-PF')

_UNWANTED = (u'SAVE', u'CITES', u'SHARIA', u'IRAN', u'WHO',
             u'CAN', u'SAO', u'IT', u'POSITION')

def run_update(in_dir, predicate=None):
    acronyms = set(_ACRONYMS)
    subjects = set()
    tags = defaultdict(list)
    for cable in cables_from_directory(in_dir, predicate):
        update_acronyms(cable, acronyms)
        update_missing_subjects(cable, subjects)
        update_tags(cable, tags)
    return {'acronyms': acronyms, 'subjects': subjects, 'tags': tags}

def update_acronyms(cable, acronyms):
    if not cable.subject:
        return
    subject_words = [w for w in cable.subject.upper().split() if not w.startswith('XXXXX')]
    acronyms.update((ac for ac in _AC_PATTERN.findall(cable.content) if ac in subject_words and ac not in _UNWANTED))

def update_missing_subjects(cable, cable_refs):
    if not parse_subject(cable.content, fix_subject=False):
        cable_refs.add(cable.reference_id)

def update_tags(cable, tags):
    for tag in cable.tags:
        if len(tag) > 5 and tag not in _KNOWN_TAGS:
            tags[tag].append(cable.reference_id)
 
def _filename(name):
    return os.path.join(os.path.dirname(__file__), name)

def _file_in_core(name):
    return os.path.join(os.path.dirname(__file__), '..', 'cablemap.core', 'cablemap/core/', name)

def _file_as_set(filename):
    if not os.path.exists(filename):
        return set()
    f = codecs.open(filename, 'rb', 'utf-8')
    s = set((l.rstrip() for l in f))
    f.close()
    return s

def _write_set(filename, s):
    f = codecs.open(filename, 'wb', 'utf-8')
    for item in sorted(s):
        f.write(item)
        f.write('\n')
    f.close()

def update_file(filename, found):
    if not found:
        return
    current = _file_as_set(filename)
    diff = current ^ found
    if diff:
        _write_set(filename, found)
        removed = current - found
        fn = os.path.basename(filename)
        header = 'Changes in %s' % fn
        print('\n' + header)
        print('=' * len(header))
        if removed:
            print('Removed:\n%r\n' % (removed))
        added = found - current
        if added:
            print('Added:\n%r\n' % (added))


_FILE_ACRONYMS = _file_in_core('acronyms.txt')
_FILE_SUBJECTS = _filename('no_subject.txt')
    
if __name__ == '__main__':
    if not os.path.isdir('./cable/'):
        raise Exception('Expected a directory "cable"')
    res = run_update('./cable/')
    update_file(_FILE_ACRONYMS, res['acronyms'])
    update_file(_FILE_SUBJECTS, res['subjects'])
    print('Valid TAGs?')
    tags = res['tags']
    for k in sorted(tags.keys()):
        print(u'%s:\n    %s' % (k, u', '.join(tags[k])))
{'fraction_non_alphanumeric': 0.19822464027489442, 'fraction_numerical': 0.0019328513136230225, 'mean_word_length': 2.925259904467547, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276340', 'n_tokens_mistral': 5957, 'n_tokens_neox': 5811, 'n_words': 1179}