<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import fec_crude_csv, doctest, cgitb

def records(inputstring):
    return list(fec_crude_csv.readstring(inputstring))

def test_endtext():
    """
    >>> records(filing_207928)
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{...'fec_ver': '5.3'...},
     {...'filer_id': 'C00410761', 'original_data': {...}...
      'committee': 'Castor for Congress'...}]
    >>> records(truncated_filing)       # same
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{...'fec_ver': '5.3'...},
     {...'filer_id': 'C00410761', 'original_data': {...}...
      'committee': 'Castor for Congress'...}]
    """

# Note the extra quote mark on the [ENDTEXT] line.
filing_207928 = '''"HDR","FEC","5.3","NGP Campaign Office(R)","1.0e","","","0",""
"F99","C00410761","Castor for Congress","P.O. Box 5419","","Tampa","FL","33675","<NAME>","20060322","MST"
[BEGINTEXT]
March 22, 2006

<NAME>
Senior Campaign Finance Analyst
Federal Elections Commission
999 E Street, NW
Washington, DC 20463

Identification Number: C00410761

Reference: Year End Report (10/1/05-12/31/05)

Dear <NAME>,

	This letter is in response to the FEC request for additional information dated February 23, 2006.  As instructed in your letter, we have amended our Year End Report and corrected the discrepancy on Line 2 of FEC Form 3Z-1.

	If you need further assistance, please contact me at (813)251-5094.


Sincerely,



<NAME>, Treasurer
Castor for Congress

[ENDTEXT]"
'''

# This is an invalid filing but it shouldn’t hang the importer.  (It
# did at one point; it would loop endlessly looking for [ENDTEXT].)
truncated_filing = '''"HDR","FEC","5.3","NGP Campaign Office(R)","1.0e","","","0",""
"F99","C00410761","Castor for Congress","P.O. Box 5419","","Tampa","FL","33675","<NAME>","20060322","MST"
[BEGINTEXT]
March 22, 2006
'''

def test_v2_02_data():
    """We can’t handle version 2.x data yet, so we should just skip
    it until we find a spec for it.  Raising an exception is not
    acceptable.

    >>> records(filing_30458_truncated)
    []
    """

filing_30458_truncated = '''/* Header
FEC_Ver_# = 2.02
Soft_Name = Vocus, Inc. PACPRO
Soft_Ver# = 6.16.040
DEC/NODEC = DEC
Date_Fmat = CCYYMMDD
NameDelim = ^
Form_Name = F3XN
FEC_IDnum = C00238725
Committee = National Air Traffic Controllers Association PAC
Schedule_Counts:
SA11ai   = 00002
SB23     = 00023
/* End Header
"F3XN","C00238725","National Air Traffic Controllers Association PAC","1325 Massachusetts Ave., NW","","Washington","DC","20005","","X","M3","",,"",20020201,20020228,290018.35,45558.46,335576.81,23000,312576.81,0,0,332,45226.46,45558.46,0,0,45558.46,0,0,0,0,0,0,0,45558.46,45558.46,0,0,0,0,0,23000,0,0,0,0,0,0,0,0,0,23000,23000,45558.46,0,45558.46,0,0,0,250065.32,2002,91011.49,341076.81,28500,312576.81,534,90477.49,91011.49,0,0,91011.49,0,0,0,0,0,0,0,91011.49,91011.49,0,0,0,0,0,28500,0,0,0,0,0,0,0,0,0,28500,28500,91011.49,0,91011.49,0,0,0,"Mr. <NAME>",20020323
"SA11ai","C00238725","IND","Glasserman^John^Mr.^Glasserman","4 Spruce Lane","","Essex Junction","VT","054524387","","","Federal Aviation Administration","Air Traffic Controller",404,,202,"15","","","","","","","","","","","","","","","Payroll Deduction ($101.00 Biweekly)","","10000072200300002"
'''

def test_windows_1252_characters():
    r"""The FEC claims that filings containing non-ASCII characters will be<commit_msg>oops, change the FEC tests to expect the FilingFormatNotDocumented exception<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import fec_crude_csv, doctest, cgitb

def records(inputstring):
    return list(fec_crude_csv.readstring(inputstring))

def test_endtext():
    """
    >>> records(filing_207928)
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{...'fec_ver': '5.3'...},
     {...'filer_id': 'C00410761', 'original_data': {...}...
      'committee': 'Castor for Congress'...}]
    >>> records(truncated_filing)       # same
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{...'fec_ver': '5.3'...},
     {...'filer_id': 'C00410761', 'original_data': {...}...
      'committee': 'Castor for Congress'...}]
    """

# Note the extra quote mark on the [ENDTEXT] line.
filing_207928 = '''"HDR","FEC","5.3","NGP Campaign Office(R)","1.0e","","","0",""
"F99","C00410761","Castor for Congress","P.O. Box 5419","","Tampa","FL","33675","<NAME>","20060322","MST"
[BEGINTEXT]
March 22, 2006

<NAME>
Senior Campaign Finance Analyst
Federal Elections Commission
999 E Street, NW
Washington, DC 20463

Identification Number: C00410761

Reference: Year End Report (10/1/05-12/31/05)

Dear <NAME>,

	This letter is in response to the FEC request for additional information dated February 23, 2006.  As instructed in your letter, we have amended our Year End Report and corrected the discrepancy on Line 2 of FEC Form 3Z-1.

	If you need further assistance, please contact me at (813)251-5094.


Sincerely,



<NAME>, Treasurer
Castor for Congress

[ENDTEXT]"
'''

# This is an invalid filing but it shouldn’t hang the importer.  (It
# did at one point; it would loop endlessly looking for [ENDTEXT].)
truncated_filing = '''"HDR","FEC","5.3","NGP Campaign Office(R)","1.0e","","","0",""
"F99","C00410761","Castor for Congress","P.O. Box 5419","","Tampa","FL","33675","<NAME>","20060322","MST"
[BEGINTEXT]
March 22, 2006
'''

def test_v2_02_data():
    '''We can’t handle version 2.x data yet, so we should just skip
    it until we find a spec for it, by means of raising an exception.

    >>> records(filing_30458_truncated)
    Traceback (most recent call last):
    ...
    FilingFormatNotDocumented: /* Header
    FEC_Ver_# 
    '''

filing_30458_truncated = '''/* Header
FEC_Ver_# = 2.02
Soft_Name = Vocus, Inc. PACPRO
Soft_Ver# = 6.16.040
DEC/NODEC = DEC
Date_Fmat = CCYYMMDD
NameDelim = ^
Form_Name = F3XN
FEC_IDnum = C00238725
Committee = National Air Traffic Controllers Association PAC
Schedule_Counts:
SA11ai   = 00002
SB23     = 00023
/* End Header
"F3XN","C00238725","National Air Traffic Controllers Association PAC","1325 Massachusetts Ave., NW","","Washington","DC","20005","","X","M3","",,"",20020201,20020228,290018.35,45558.46,335576.81,23000,312576.81,0,0,332,45226.46,45558.46,0,0,45558.46,0,0,0,0,0,0,0,45558.46,45558.46,0,0,0,0,0,23000,0,0,0,0,0,0,0,0,0,23000,23000,45558.46,0,45558.46,0,0,0,250065.32,2002,91011.49,341076.81,28500,312576.81,534,90477.49,91011.49,0,0,91011.49,0,0,0,0,0,0,0,91011.49,91011.49,0,0,0,0,0,28500,0,0,0,0,0,0,0,0,0,28500,28500,91011.49,0,91011.49,0,0,0,"Mr. <NAME>",20020323
"SA11ai","C00238725","IND","Glasserman^John^Mr.^Glasserman","4 Spruce Lane","","Essex Junction","VT","054524387","","","Federal Aviation Administration","Air Traffic Controller",404,,202,"15","","","","","","","","","","","","","","","Payroll Deduction ($101.00 Biweekly)","","10000072200300002"
'''

def test_windows_1252_characters():
    r"""The FEC claims that filings containing non-ASCII characters will be
{'fraction_non_alphanumeric': 0.19640287769784173, 'fraction_numerical': 0.1623021582733813, 'mean_word_length': 5.20625, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '953803', 'n_tokens_mistral': 3349, 'n_tokens_neox': 2658, 'n_words': 564}