<commit_before># -*- coding: utf-8 -*-
"""
Created on Mon May 15 12:11:35 2017

@author: MudassirEX
"""

import requests
import re
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
import datetime

def find_in_list(name_list, name):
        full_name = [l for l in name_list for m in [re.search(name, l, flags=re.IGNORECASE)] if m]
        if len(full_name) > 1:
            return(name+': Enter Manually: '+'/'.join(full_name))
        if len(full_name) == 0:
            return(name+': Enter Manually')
        return full_name[0]

# Read in match data
all_matches = pd.read_excel('convert.xlsx')
all_matches = all_matches[pd.isnull(all_matches['note'])]

for index, match in all_matches.iterrows():
    q = 1
    #### Scorecard details
    link = match['scorecard']
    page = requests.get(link)
    soup = BeautifulSoup(page.content, 'lxml')
    <commit_msg>Remove checking of empty note column<commit_after># -*- coding: utf-8 -*-
"""
Created on Mon May 15 12:11:35 2017

@author: MudassirEX
"""

import requests
import re
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
import datetime

def find_in_list(name_list, name):
        full_name = [l for l in name_list for m in [re.search(name, l, flags=re.IGNORECASE)] if m]
        if len(full_name) > 1:
            return(name+': Enter Manually: '+'/'.join(full_name))
        if len(full_name) == 0:
            return(name+': Enter Manually')
        return full_name[0]

# Read in match data
all_matches = pd.read_excel('convert.xlsx')

for index, match in all_matches.iterrows():
    q = 1
    #### Scorecard details
    link = match['scorecard']
    page = requests.get(link)
    soup = BeautifulSoup(page.content, 'lxml')
    
{'fraction_non_alphanumeric': 0.1250723798494499, 'fraction_numerical': 0.020845396641574986, 'mean_word_length': 3.277227722772277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2695820', 'n_tokens_mistral': 607, 'n_tokens_neox': 576, 'n_words': 175}