<jupyter_start><jupyter_text>Reading in three data sets<jupyter_code>fda=pd.read_csv("../data/working/fda_clean.csv")
ndc=pd.read_csv("../data/working/cleanNDC.csv")
#dna=pd.read_csv("../data/working/dna_clean.csv")
dna.isnull().values.any()
dna.isnull().sum()
fda.info()
ndc.info()
dna.info()
dna=dna.dropna()
dna.isnull().values.any()
dna.isnull().sum()
dna.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 63954 entries, 0 to 64004
Data columns (total 5 columns):
 #   Column             Non-Null Count  Dtype 
---  ------             --------------  ----- 
 0   Unnamed: 0         63954 non-null  int64 
 1   Unnamed: 0.1       63954 non-null  int64 
 2   Code               63954 non-null  object
 3   Description        63954 non-null  object
 4   cleaned_companies  63954 non-null  object
dtypes: int64(2), object(3)
memory usage: 2.9+ MB<jupyter_text>Creating a function to measure fuzz ratio between two data sets<jupyter_code>#df1=fda.iloc[:100]
#df2=ndc.iloc[:100]
#df3=dna.iloc[:100]
#Function that creates a matrix of fuzzy scores
#l1 and l2 are the lists
#nrows will be the number of rows you will need in the matrix
#ncols will be the number of columns you will need

#distribution of the matrix
def matrix_score(list1, list2, nrows, ncols):
    #creates a matrix of zeros with the dimensions specified from the parameters
    #So if you had a list that had 5 elements in it, your matrix would be a 5x5 matrix; nrows would be 5 and ncols would be 5
    matrix = np.zeros(shape = (nrows, ncols))
   
    #initialize row and col values to 0
    row = 0
    col = 0
   
    for i in list1:
        if row % (len(list1)//10) == 0:
            print("progress: %.2f" % (100*row/len(list1)) + "%")
        for j in list2:
            score = fuzz.ratio(i,j)
            matrix[row][col] = score
            col+=1
        col = 0
        row+=1
    #savetxt('/sfs/lustre/bahamut/scratch/sr2wm/Git/dspg20BI/data/working/dna_matrix_1c.csv', matrix, delimiter=',')
    save('/sfs/lustre/bahamut/scratch/sr2wm/Git/dspg20BI/data/working/dna_matrix_1b.npy', matrix)
    return matrix
#data1=dna[:10659]
data2=dna[10659:21318]
#data.info()
matrix=matrix_score(data2['cleaned_companies'].tolist(), dna['cleaned_companies'].tolist(), 10659, 63954)
d1=np.load('../data/working/dna_matrix_1b.npy')
d=np.load('../data/working/dna_matrix_1a.npy')
dna_matrix=vstack([d,d1])
dna_matrix
d1
d
save('/sfs/lustre/bahamut/scratch/sr2wm/Git/dspg20BI/data/working/dna_matrix_1.npy', dna_matrix)
data_matrix=np.load('../data/working/dna_matrix_1.npy')
data_matrix
#This function takes the matrix and list of cleaned company names as arguments and returns a dataframe representing the martrix passed
def convertMatrixToDf(matrix, ls):
    #In order for the code to work, the matrix passed must first be converted from float to int
    matrix = matrix.astype(int)
    #Then the matrix needs to be converted to str
    matrix = matrix.astype(str)
    cleaned_matrix = [[int(item.split()[0]) for item in row] for row in matrix]
    matrix_scores = pd.DataFrame(cleaned_matrix, columns = ls)
    #Since there are no company name rows, this function makes them
    matrix_scores['companies'] = ls
   
    #returns the matrix as a dataframe, with type int
    return matrix_scores.astype(int)
ls=data['cleaned_companies'].tolist()
df=convertMatrixToDf(matrix, ls)
df.to_csv(r'/sfs/lustre/bahamut/scratch/sr2wm/Git/dspg20BI/data/working/dna_matrix_1a.csv')
dna_data_1b=pd.read_csv("../data/working/dna_matrix_1b.csv")
dna_data_1b.head()
col_name= []
for row in dna_data.itertuples():
    for col in row:
        if row[col]== 100:
        col_name.append(col.name)
def matchWithFuzzyNames(list1, list2, original):
    match = pd.DataFrame(columns = ['original company names', 'clean company name', 'company matches', 'fuzz ratio'])
    for i in list1:
        matches = []
        score = []
        score_partial = []
        for j in list2:
            if (fuzz.ratio(i,j)) == 100:
                matches.append(j)
                score.append(fuzz.ratio(i,j))
                score_partial.append((fuzz.partial_ratio(i,j)))
        match = match.append({ 'clean company name': i,'company matches': matches, 'fuzz ratio': score, 'fuzz partial ratio': score_partial},ignore_index=True)
        
    match['original company names'] = original
        


    return match

#matching=match.head(len(match))
matching=match.head(500)<jupyter_output><empty_output><jupyter_text>Specify the col name for each dataset<jupyter_code>matchWithFuzzyNames(fda, ndc, fda.col_name)
ndc_matching['clean_company_name'.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12865748709122204, 'fraction_numerical': 0.034853700516351116, 'mean_word_length': 3.4701923076923076, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '372456', 'license': 'MIT', 'n_tokens_mistral': 1762, 'n_tokens_neox': 1642, 'n_words': 403, 'path': 'src/Matching/.ipynb_checkpoints/Delete_matchingFunction-checkpoint.ipynb', 'repo_name': 'DSPG-Young-Scholars-Program/dspg20BI', 'size': '4648'}