<jupyter_start><jupyter_text>Everything you need to know about dataset !!!  Reading and Pre-processing datasets<jupyter_code>train_files = os.listdir('Training Dataset')
test_files = os.listdir('Test Data')
result = pd.read_csv('Results2394dca.csv')
pension = pd.read_excel('pension_good.xlsx')
rules = pd.read_excel('rules.xlsx')
# sample_data = pd.read_csv('')
rules.head()
result.head()
result.columns
result.columns = ['For Lookup', 'Year', 'Credit Name', 'State', 'Security ID',
       'Org ID', 'FYE', 'Accounting Basis', 'Pension Plan 1 Name',
       'Pension Plan 1 Measurement Date',
       'Pension Plan 1 Total Pension Liability', 'Balance Sheet Cash',
       'Pension Plan 2 Name', 'Pension Plan 2 Measurement Date',
       'Pension Plan 2 Total Pension Liability']<jupyter_output><empty_output><jupyter_text>result_pp = pp.ProfileReport(result)result_pp.to_file('result.html')pdfkit.from_file('result.html', 'result.pdf') pension_pp = pp.ProfileReport(pension)pension_pp.to_file('pension.html')pdfkit.from_file('pension.html', 'pension.pdf') rules_pp = pp.ProfileReport(rules)rules_pp.to_file('rules.html')pdfkit.from_file('rules.html', 'rules.pdf')<jupyter_code>result.head()
rules.head()
train_files
sorted(test_files)
test_files.remove('.ipynb_checkpoints')
data = list(map(lambda x : x.strip().lower(), [x for x in parser.from_file('Test Data/' + test_files[0])['content'].split('\n') if x!='' and x!=' ']))
'basis' in data
for i in data:
    if 'net pension liability' in i:
        print(i)
        
# for i in range(len(data)):
#     if 'pension plan' in data[i]:
#         print(i)
account= ''
expense = ''<jupyter_output><empty_output><jupyter_text>First Detecting Account<jupyter_code>check = test_files[0]
for i in data:
    if 'basis of accounting' in i:
        tmp = i.find('basis of accounting')
        first = 0; last = 0;
        if tmp - 50<0: #Checking for desired keyword in first and last 50 characters from the tmp
            first = 0
        else:
            first = tmp - 50
        
        if tmp+50 > len(i)-1:
            last = len(i) - 1
        else:
            last = tmp+50
            
        if 'modified accrual' in i[first:tmp] or 'modified accrual' in i[tmp:last]:
            account = 'Modified Accrual'
            break
        elif 'modified cash' in i[first:tmp] or 'modified cash' in i[tmp:last]:
            account = 'Modified Cash'
            break
        elif 'accrual' in i[first:tmp] or 'accrual' in i[tmp:last]:
            account = 'Accrual'
            break
        elif 'cash' in i[first:tmp] or 'cash' in i[tmp:last]:
            account = 'Cash'
            break
        elif 'regulatory' in i[first:tmp] or 'regulatory' in i[tmp:last]:
            account = 'Regulatory'
            break
        else:
            account = '0'
print(account)<jupyter_output>Accrual<jupyter_text>Now detecting Balance Sheet Cash<jupyter_code>stack = []
upto = 0
bills = []; curr = [];

def isblank (myString):
    if myString and myString.strip():
        #myString is not None AND myString is not empty or blank
        return False
    #myString is None OR myString is empty or blank
    return True

#For extracting money from the possible section of document
def amount(stack):
    temp = 0
    target = ''
    stack = list(map(lambda x: x.split(' '), stack))
    for i in range(len(stack)):
        stack[i] = list(map(lambda x: x.strip(string.punctuation+' '), stack[i]))
        stack[i] = ' '.join(stack[i]).split() #For removing extra blank characters
        for j in range(len(stack[i])):
            if len(stack[i][j].split(',')) > 1:
                var = stack[i][j].split(',')
                for k in range(len(var)):
                    try:
                        int(var[k])
                        target =  target + var[k] #Trying to avoid using spacy for detecting number here, for unknown scenario reasons
                    except:
                        break
                                               
                        
        if isblank(target):
            pass
        else:
            return int(target)





for i in range(len(data)-1):
#     print(i)
    #First detecting if we've all the 4 keywords
    if 'balance' in data[i]:
        if 'sheet' in data[i]:
            if 'government' in data[i]:
                if 'fund' in data[i]:
                    stack = data[i:i+30]
                    expense = amount(stack)
#                     print("Expense", expense)
                    if expense != None and expense != 0:
#                         print('Four executing')
                        del data[i:i+30]
                        break
                    else:
                        pass
    elif 'balance sheet' in data[i]:
#         print("Second executing")
        expense = amount(stack)
        if expense != None and expense != 0:
            del data[i:i+30]
            break
        else:
            pass
print(expense)<jupyter_output>182009<jupyter_text>Now Pension plan 1,2 name<jupyter_code>nlp = spacy.load('en_core_web_sm')
b1name = None; b2name =  None; del1 = 0
b1amt = ''; b2amt = '';
b1date =  None; b2date =  None;
doc = nlp("Next week I'll   be in Madrid, which means on 1/12/22, 2019")
for token in doc:
    print("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}".format(
        token.text,
        token.idx,
        token.lemma_,
        token.is_punct,
        token.is_space,
        token.shape_,
        token.pos_,
        token.tag_        
    ))
    
for ent in doc.ents:
    if ent.label_ == 'DATE':
        print(ent.text)
        break
','.join(data[0:5])
data = list(map(lambda x : x.strip().lower(), [x for x in parser.from_file('Test Data/' + test_files[0])['content'].split('\n') if x!='' and x!=' ']))<jupyter_output><empty_output><jupyter_text>Pension 1<jupyter_code>for i in range(len(data)): # This is for detecting pension plan1 name and date
#     print(i)
    if 'pension plan' in data[i]:
#         print(data[i])
        tmp = data[i].split(' ')
        tmp = ' '.join(tmp).split() 
#         print(tmp)
#         print('*'*100)

        for j in range(len(tmp)):
            if tmp[j]=='plan':
                doc = nlp(data[i])
                for token in range(len(doc)):
                    if doc[token].text == 'plan':
                        try:
#                             print(doc[token+1])
#                             print(token, len(doc), "im doing exp")
                            if doc[token+1].pos_=='NOUN':
                                b1name = doc[token+1].text
                                stack = ','.join(data[i:i+10])
                                test = nlp(str(stack))
                                for ent in test.ents:
                                    if ent.label_ =='DATE':
                                        b1date = ent.text
                                        break
                                
                                
                        except:
                            print("next token was over the top")
               

    if b1name != None and b1date != None:
        print("Deleting", i, "to", i+10) #Because don't wanna use it again!!
        del data[i:i+10]
        break

# Now to detect pension plan 1 number
large = -100
todel = 0
b1amt = ''
for i in range(len(data)):
    if 'net' in data[i]:
        if 'pension' in data[i]:
            if 'liability' in data[i]:
               
#                 print(data[i])
#                 print('*' * 100)
                tmp = data[i].split(' ')
                tmp = ' '.join(tmp).split() 
#                 print(" wanna see tmp too", tmp)
                for j in range(len(tmp)):
                    b1amt = ''
#                     if len(tmp[j].split(',')) > 1:                     # if len(' '.join('liability,'.split(',')).split()) > 1:
                    if len(' '.join(tmp[j].split(',')).split()) > 1:
#                         print(tmp[j])
#                         print('*' * 100)
                        broke = ' '.join(tmp[j].split(',')).split()
                        
#                         print("This is broke", broke)
#                         print('~' * 100)
                        for k in range(len(broke)):
                            try:
                                int(broke[k])
#                                 print("Int happened")
                                b1amt = b1amt + broke[k]
#                                 print("This is b1amt", b1amt)
                            except:
                                break
                        try:
                            if float(b1amt) > large:
                                large = int(b1amt)
                                todel = i
                        except:
                            print("This instance, wasn't good", b1amt)


b1amt = large
del data[todel]
        
        
print("This is the pension plan 1 name", b1name)
print("This is the pension plan 1 date", b1date)
print("This is the net liability money", b1amt)
print("Todel deleted was: ",todel)<jupyter_output>next token was over the top
Deleting 1948 to 1958
This instance, wasn't good 
This is the pension plan 1 name investments
This is the pension plan 1 date december 31
This is the net liability money 342748
Todel deleted was:  1722<jupyter_text>Pension 2<jupyter_code>for i in range(len(data)): # This is for detecting pension plan1 name and date
    if 'pension plan' in data[i]:
        tmp = data[i].split(' ')
        tmp = ' '.join(tmp).split() 
        for j in range(len(tmp)):
            if tmp[j]=='plan':
                doc = nlp(data[i])
                for token in range(len(doc)):
                    if doc[token].text == 'plan':
                        try:
                            if doc[token+1].pos_=='NOUN':
                                b2name = doc[token+1].text
                                stack = ','.join(data[i:i+10])
                                test = nlp(str(stack))
                                for ent in test.ents:
                                    if ent.label_ =='DATE':
                                        b2date = ent.text
                                        break

                        except:
                            print("next token was over the top")
               

    if b2name != None and b2date != None:
        print("Deleting", i, "to", i+10) #Because don't wanna use it again, for pension plan 2!!
        del data[i:i+10]
        break

# Now to detect pension plan 1 number
large = -100
todel = 0
b2amt = ''
for i in range(len(data)):
    if 'net' in data[i]:
        if 'pension' in data[i]:
            if 'liability' in data[i]:
                tmp = data[i].split(' ')
                tmp = ' '.join(tmp).split() 
                for j in range(len(tmp)):
                    b2amt = ''
                    if len(' '.join(tmp[j].split(',')).split()) > 1:
                        broke = ' '.join(tmp[j].split(',')).split()
                        for k in range(len(broke)):
                            try:
                                int(broke[k])
                                b2amt = b2amt + broke[k]
                            except:
                                break
                        try:
                            if float(b2amt) > large:
                                large = int(b2amt)
                                todel = i
                        except:
                            print("This instance, wasn't good", b2amt)


b2amt = large
        
        
print("This is the pension plan 2 name", b2name)
print("This is the pension plan 2 date", b2date)
print("This is the net liability money", b2amt)<jupyter_output>next token was over the top
Deleting 1994 to 2004
This instance, wasn't good 
This is the pension plan 2 name investments
This is the pension plan 2 date the year ended december 31, 2018
This is the net liability money 180215<jupyter_text>Finally combining all of em into one function, and applying for all the test data set<jupyter_code>len(os.listdir('Test Data/'))
pdf_names = test_files.copy()
test_files = list(map(lambda x: x.split('_')[0], test_files))
minn = 1000
array = np.zeros(len(test_files))
for i in range(len(test_files)):
    minn = 1000
    for j in range(len(result['Credit Name'])):
        if nltk.edit_distance(result.iloc[j]['Credit Name'], test_files[i]) < minn:
            minn = nltk.edit_distance(result.iloc[j]['Credit Name'], test_files[i]) 
            array[i] = j
    
    
account
for index in range(len(test_files)):
    try:
        data = list(map(lambda x : x.strip().lower(), [x for x in parser.from_file('Test Data/' + pdf_names[index])['content'].split('\n') if x!='' and x!=' ']))
        
        account= ''
        expense = ''
        #First detecting basis of accounting
        for i in data:
            if 'basis of accounting' in i:
                tmp = i.find('basis of accounting')
                first = 0; last = 0;
                if tmp - 50<0: #Checking for desired keyword in first and last 50 characters from the tmp
                    first = 0
                else:
                    first = tmp - 50

                if tmp+50 > len(i)-1:
                    last = len(i) - 1
                else:
                    last = tmp+50

                if 'modified accrual' in i[first:tmp] or 'modified accrual' in i[tmp:last]:
                    account = 'Modified Accrual'
                    break
                elif 'modified cash' in i[first:tmp] or 'modified cash' in i[tmp:last]:
                    account = 'Modified Cash'
                    break
                elif 'accrual' in i[first:tmp] or 'accrual' in i[tmp:last]:
                    account = 'Accrual'
                    break
                elif 'cash' in i[first:tmp] or 'cash' in i[tmp:last]:
                    account = 'Cash'
                    break
                elif 'regulatory' in i[first:tmp] or 'regulatory' in i[tmp:last]:
                    account = 'Regulatory'
                    break
                else:
                    account = '0'
                    
        #Now tryna detect Bank balance governmental funds
        stack = []
        upto = 0
        bills = []; curr = [];

        def isblank (myString):
            if myString and myString.strip():
                #myString is not None AND myString is not empty or blank
                return False
            #myString is None OR myString is empty or blank
            return True

        #For extracting money from the possible section of document
        def amount(stack):
            large = 0.0
            target = ''
            stack = list(map(lambda x: x.split(' '), stack))
            for i in range(len(stack)):
                stack[i] = list(map(lambda x: x.strip(string.punctuation+' '), stack[i]))
                stack[i] = ' '.join(stack[i]).split() #For removing extra blank characters

                for j in range(len(stack[i])):
                    if len(stack[i][j].split(',')) > 1:
                        target = ''
                        var = stack[i][j].split(',')
                        for k in range(len(var)):
                            try:
                                int(var[k])
                                target =  target + var[k] #Trying to avoid using spacy for detecting number here, for unknown scenario reasons
                            except:
                                break
                        try:
                            if int(target) > int(large):
                                large = target
                        except:
                            pass
            return int(large)





        for i in range(len(data)-1):
        #     print(i)
            #First detecting if we've all the 4 keywords
            if 'balance' in data[i]:
                if 'sheet' in data[i]:
                    if 'government' in data[i]:
                        if 'fund' in data[i]:
                            stack = data[i:i+30]
                            expense = amount(stack)
        #                     print("Expense", expense)
                            if expense != None and expense != 0:
        #                         print('Four executing')
                                del data[i:i+30]
                                break
                            else:
                                pass
            elif 'balance sheet' in data[i]:
        #         print("Second executing")
                expense = amount(stack)
                if expense != None and expense != 0:
                    del data[i:i+30]
                    break
                else:
                    pass
                
                
        b1name = None; b2name =  None; del1 = 0
        b1amt = ''; b2amt = '';
        b1date =  None; b2date =  None;
        
        for i in range(len(data)): # This is for detecting pension plan1 name and date
            if 'pension plan' in data[i]:
                tmp = data[i].split(' ')
                tmp = ' '.join(tmp).split() 
                for j in range(len(tmp)):
                    if tmp[j]=='plan':
                        doc = nlp(data[i])
                        for token in range(len(doc)):
                            if doc[token].text == 'plan':
                                try:
                                    if doc[token+1].pos_=='NOUN':
                                        b1name = doc[token+1].text
                                        stack = ','.join(data[i:i+10])
                                        test = nlp(str(stack))
                                        for ent in test.ents:
                                            if ent.label_ =='DATE':
                                                b1date = ent.text
                                                break
                                except:
                                    pass
#                                     print("next token was over the top")


            if b1name != None and b1date != None:
#                 print("Deleting", i, "to", i+10) #Because don't wanna use it again!!
                del data[i:i+10]
                break

        # Now to detect pension plan 1 number
        large = -100
        todel = 0
        b1amt = ''
        for i in range(len(data)):
            if 'net' in data[i]:
                if 'pension' in data[i]:
                    if 'liability' in data[i]:
                        tmp = data[i].split(' ')
                        tmp = ' '.join(tmp).split() 
                        for j in range(len(tmp)):
                            b1amt = ''
                            if len(' '.join(tmp[j].split(',')).split()) > 1:
                                broke = ' '.join(tmp[j].split(',')).split()
                                for k in range(len(broke)):
                                    try:
                                        int(broke[k])
                                        b1amt = b1amt + broke[k]
                                    except:
                                        break
                                try:
                                    if float(b1amt) > large:
                                        large = int(b1amt)
                                        todel = i
                                except:
                                    pass
#                                     print("This instance, wasn't good", b1amt)


        b1amt = large
        if b1amt is None or b1amt == -100 or b1amt is '':
            b1amt = 0
        del data[todel]
        
         
        for i in range(len(data)): # This is for detecting pension plan2 name and date
            if 'pension plan' in data[i]:
                tmp = data[i].split(' ')
                tmp = ' '.join(tmp).split() 
                for j in range(len(tmp)):
                    if tmp[j]=='plan':
                        doc = nlp(data[i])
                        for token in range(len(doc)):
                            if doc[token].text == 'plan':
                                try:
                                    if doc[token+1].pos_=='NOUN':
                                        b2name = doc[token+1].text
                                        stack = ','.join(data[i:i+10])
                                        test = nlp(str(stack))
                                        for ent in test.ents:
                                            if ent.label_ =='DATE':
                                                b2date = ent.text
                                                break

                                except:
                                    pass
#                                     print("next token was over the top")


            if b2name != None and b2date != None:
#                 print("Deleting", i, "to", i+10) #Because don't wanna use it again, for pension plan 2!!
                del data[i:i+10]
                break
                
        # Now to detect pension plan 2 amount
        large = -100
        todel = 0
        b2amt = ''
        for i in range(len(data)):
            if 'net' in data[i]:
                if 'pension' in data[i]:
                    if 'liability' in data[i]:
                        tmp = data[i].split(' ')
                        tmp = ' '.join(tmp).split() 
                        for j in range(len(tmp)):
                            b2amt = ''
                            if len(' '.join(tmp[j].split(',')).split()) > 1:
                                broke = ' '.join(tmp[j].split(',')).split()
                                for k in range(len(broke)):
                                    try:
                                        int(broke[k])
                                        b2amt = b2amt + broke[k]
                                    except:
                                        break
                                try:
                                    if float(b2amt) > large:
                                        large = int(b2amt)
                                        todel = i
                                except:
                                    pass
#                                     print("This instance, wasn't good", b2amt)
        if b2name is None or b2name == '':
            b2name = 'Not Found'
            
        if b2date is None or b2date == '':
            b2date = 'No Date'


        b2amt = large 
        if b2amt is None or b2amt == -100 or b2amt is '':
            b2amt = 0
        
        result['Accounting Basis'][array[index]] = account
        result['Balance Sheet Cash'][array[index]] = expense
        result['Pension Plan 1 Name'][array[index]] = b1name
        result['Pension Plan 1 Total Pension Liability'][array[index]] = b1amt
        result['Pension Plan 1 Measurement Date'] = b1date
        result['Pension Plan 2 Name'][array[index]] = b2name
        result['Pension Plan 2 Measurement Date'][array[index]] = b2date
        result['Pension Plan 2 Total Pension Liability'][array[index]] = b2amt

        
    except:
        print("Pdf file was image: ", pdf_names[index])
copy = result.copy()
result.fillna(method='ffill', inplace=True)
result.fillna(method='bfill', inplace=True)
result.fillna('0', inplace=True)
result.to_csv('results.csv', index=None)
result
for i in range(len(test_files)):
    print("Matching done was: ", test_files[i], "to", result.iloc[int(array[i])]['Credit Name'])
array[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1102189164060519, 'fraction_numerical': 0.01458033978530978, 'mean_word_length': 1.180921658986175, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '218037', 'license': 'MIT', 'n_tokens_mistral': 6650, 'n_tokens_neox': 6241, 'n_words': 1803, 'path': 'crisil.ipynb', 'repo_name': 'shivam13juna/crisil_challenge_HE', 'size': '23662'}