<commit_before>'''
Checks the titles in a 360Giving data file (csv) against the titles
set out in the schema.

Uses the example csv file from:
http://docs.threesixtygiving.org/assets/standard/schema/summary-table/360-giving-schema-titles.csv/Activity.csv
renamed as 360Activity.csv, and in the same directory as this script

Usage: python check_titles.py

You might want to pipe the results to a file:
python check_titles.py > results.txt
'''

import csv
import os
#import pandas as pd

# Required fields as defined here:
# http://docs.threesixtygiving.org/docs/#toc5
required_fields = ['Identifier',
                    'Title',
                    'Description',
                    'Currency',
                    'Amount Awarded',
                    'Award Date',
                    'Recipient Org:Identifier',
                    'Recipient Org:Name',
                    'Funding Org:Identifier',
                    'Funding Org:Name'
                  ]

# Get a list of titles used in the example csv file
with open('360Activity.csv', 'rb') as csvfile:
      reader = csv.reader(csvfile, delimiter=',', quotechar='"')
      titles = next(reader)
      
#df = pd.read_csv('360Activity.csv')
#print (df)
#print df['Identifier'].count()
#print (list(df))


# Find all csv files in the current directory
# Ignore the example csv file tho!
csvs = []
for i in os.listdir(os.getcwd()):
    if i.endswith(".csv") and i != '360Activity.csv': 
        csvs.append(i)
        continue
    else:
        continue


'''
Loop through all the csv files and compare the column headings with
the example and with required fields.

Uses the python 'set' function to compare lists
'''
for csv_file in csvs:
    # print ('Checking ' + csv_file)
    with open(csv_file, 'rb') as csvfile:
          reader = csv.reader(csvfile, delimiter=',', quotechar='"')
          first_row = next(reader)
          #for row in spamreader:
          #   print ', '.join(row)
          # print (first_row)
          
          print('Standard fields used: ' + csv_file)<commit_msg>Adds optional command line argument so data can be kept elsewhere<commit_after>'''
Checks the titles in a 360Giving data file (csv) against the titles
set out in the schema.

Uses the example csv file from:
http://docs.threesixtygiving.org/assets/standard/schema/summary-table/360-giving-schema-titles.csv/Activity.csv
renamed as 360Activity.csv, and in the same directory as this script

Usage: 
Make sure 360Activity.csv is in the same directory as this script
If your csv files are in the same directory, simply run:

python check_titles.py

Optionally you can add a command line argument with the path to your csvs
e.g. If your files are in a 'data' directory 
python check_titles.py data

You might want to pipe the results to a file:
python check_titles.py > results.txt
'''

import csv
import os
import sys
#import pandas as pd

# Required fields as defined here:
# http://docs.threesixtygiving.org/docs/#toc5
required_fields = ['Identifier',
                    'Title',
                    'Description',
                    'Currency',
                    'Amount Awarded',
                    'Award Date',
                    'Recipient Org:Identifier',
                    'Recipient Org:Name',
                    'Funding Org:Identifier',
                    'Funding Org:Name'
                  ]

# Get a list of titles used in the example csv file
with open('360Activity.csv', 'rb') as csvfile:
      reader = csv.reader(csvfile, delimiter=',', quotechar='"')
      titles = next(reader)
      
#df = pd.read_csv('360Activity.csv')
#print (df)
#print df['Identifier'].count()
#print (list(df))


# Find all csv files in the current directory
# Ignore the example csv file tho!
if len(sys.argv) == 2:
    folder = sys.argv[1] 
else:
    folder = os.getcwd()

csvs = []
for i in os.listdir(folder):
    if i.endswith(".csv") and i != '360Activity.csv': 
        csvs.append(i)
        continue
    else:
        continue


'''
Loop through all the csv files and compare the column headings with
the example and with required fields.

Uses the python 'set' function to compare lists
'''
for csv_file in csvs:
    # print ('Checking ' + csv_file)
    with open(folder + '/' + csv_file, 'rb') as csvfile:
          reader = csv.reader(csvfile, delimiter=',', quotechar='"')
          first_row = next(reader)
          #for row in spamreader:
          #   print ', '.join(row)
          # print (first_row)
          
          print('Standard fields used: ' + csv_file)
{'fraction_non_alphanumeric': 0.10374449339207048, 'fraction_numerical': 0.00947136563876652, 'mean_word_length': 2.4690603514132925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2451516', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1302, 'n_words': 462}