<commit_before>#!/usr/bin/python
#Each function from BuildTables is a database function, and reads in .json files from the noted categories, then builds its named database table
#Each function from APIRequests is an API request function, and pulls the corresponding .json files from Bungie's server.
from buildTables import buildBungieTable
from buildTables import buildDestinyTable
from buildTables import buildStatTables
from apiRequests import getClanUserJSONs
from apiRequests import getIndividualUserJSONs
from apiRequests import getUserStatsJSONs
from Tokens.getHeader import getHeader
from Tokens.getClanId import getClanId
import sys

#Path to Clan data
clan = './Clan/'
#Path to User data
users = './Users/'
#Path to Stats data
stats = './Stats/'

#Path to database
db = './guardians.db'

#Path to header file
headerPath = './Tokens/header.txt'
#Path to ClanId file
clanIdPath = './Tokens/clanId.txt'

#Header
header = getHeader(headerPath)
#Clan Id
clanId = getClanId(clanIdPath)

#Start of function calls. Needs to follow the proper data flow to ensure that the next step always has the appropriate database or files that it needs.
def fullSetup():
    getClanUserJSONs(clan, header, clanId)     #Step 1 - Pull list of clan members
    buildBungieTable(clan, db)                 #Step 2 - Build Bungie table, listing clan members
    getIndividualUserJSONs(users, db, header)  #Step 3 - Pull user files, using Bungie table
    buildDestinyTable(users, db)               #Step 4 - Build Destiny table, listing user Ids
    getUserStatsJSONs(stats, db, header)       #Step 5 - Pull user stat files, using Destiny table
    buildStatTables(stats, db)                 #Step 6 - Build stat tables, using user stat files

def tablesOnly():
    buildBungieTable(clan, db)                 #Step 2 - Build Bungie table, listing clan members
    buildDestinyTable(users, db)               #Step 4 - Build Destiny table, listing user Ids
    buildStatTables(stats, db)                 #Step 6 - Build stat tables, using user stat files

if __name__ == "__main__":
    args = sys.argv
    if len(args) == 1:
        fullSetup()
    elif args[1] == '1':
        tablesOnly()
    else:
        fullSetup()<commit_msg>Cleaned up the million line of imports to just 2<commit_after>#!/usr/bin/python
#Each function from BuildTables is a database function, and reads in .json files from the noted categories, then builds its named database table
#Each function from APIRequests is an API request function, and pulls the corresponding .json files from Bungie's server.
import buildTables as bt
import apiRequests as ar
from Tokens.getHeader import getHeader
from Tokens.getClanId import getClanId
import sys

#Path to Clan data
clan = './Clan/'
#Path to User data
users = './Users/'
#Path to Stats data
stats = './Stats/'

#Path to database
db = './guardians.db'

#Path to header file
headerPath = './Tokens/header.txt'
#Path to ClanId file
clanIdPath = './Tokens/clanId.txt'

#Header
header = getHeader(headerPath)
#Clan Id
clanId = getClanId(clanIdPath)

#Start of function calls. Needs to follow the proper data flow to ensure that the next step always has the appropriate database or files that it needs.
def fullSetup():
    ar.getClanUserJSONs(clan, header, clanId)     #Step 1 - Pull list of clan members
    bt.buildBungieTable(clan, db)                 #Step 2 - Build Bungie table, listing clan members
    ar.getIndividualUserJSONs(users, db, header)  #Step 3 - Pull user files, using Bungie table
    bt.buildDestinyTable(users, db)               #Step 4 - Build Destiny table, listing user Ids
    ar.getUserStatsJSONs(stats, db, header)       #Step 5 - Pull user stat files, using Destiny table
    bt.buildStatTables(stats, db)                 #Step 6 - Build stat tables, using user stat files

def tablesOnly():
    bt.buildBungieTable(clan, db)                 #Step 2 - Build Bungie table, listing clan members
    bt.buildDestinyTable(users, db)               #Step 4 - Build Destiny table, listing user Ids
    bt.buildStatTables(stats, db)                 #Step 6 - Build stat tables, using user stat files

if __name__ == "__main__":
    args = sys.argv
    if len(args) == 1:
        fullSetup()
    elif args[1] == '1':
        tablesOnly()
    else:
        fullSetup()
{'fraction_non_alphanumeric': 0.07767669699090272, 'fraction_numerical': 0.005831583858175881, 'mean_word_length': 3.2329713721618956, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7181121', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1234, 'n_words': 510}