<commit_before>ifeq ($(DATABASE_ENGINE),mysql)
    DBCLIENT=mysql
else ifeq ($(DATABASE_ENGINE),sqlite)
    DBCLIENT=sqlite3
else
    DBCLIENT=psql
endif

ifeq ($(WATCHDOG_TABLE),)
    WATCHDOG_TABLE=watchdog_dev
endif

export PYTHONPATH := ../:.:$(PWD)/../vendor:$(PYTHONPATH)

PYTHON=python

CRAWL=../data/crawl
PARSE=../data/parse
LOAD=../data/load

TSVS=$(LOAD)/bill.tsv \
	 $(LOAD)/vote.tsv \
	 $(LOAD)/zip4.tsv \
	 $(LOAD)/census_meta.tsv \
	 $(LOAD)/census_data.tsv

NJS=$(PARSE)/almanac.njs \
    $(PARSE)/earmarks.njs \
    $(PARSE)/fec.njs \
    $(PARSE)/govtrack.njs \
    $(PARSE)/soi.njs \
    $(PARSE)/voteview.njs

LOADERS = .schema .json \
          .almanac .centers .voteview .govtrack \
          .bills .interests .votesmart .photos .fec .census .soi .earmarks

all: database
clean:
	rm $(LOADERS)

cleanload:
	rm $(TSVS)

cleanparse:
	rm $(NJS)

database: $(LOADERS)

.manual: .schema load/manual.py load/manual/*.json
	python load/manual.py
	touch $@

.schema: ../schema.py
	python -c "import schema; schema.sql.recreate(); schema.sql.grantall('watchdog')"
	touch $@

# not currently using .shapes
.shapes: .json load/shapes.py parse/shapes.py
	$(PYTHON) load/shapes.py
	touch $@

.almanac: .json load/almanac.py parse/almanac.py
	$(PYTHON) load/almanac.py
	touch $@

.centers: .json load/govtrack_gis.py ../data/crawl/govtrack/gis/gmapdata/*
	$(PYTHON) load/govtrack_gis.py
	touch $@

.govtrack: load/govtrack.py parse/govtrack.py ../data/crawl/govtrack/us/110/repstats/*xml .json
	$(PYTHON) load/govtrack.py
	touch $@

.voteview: .json load/voteview.py parse/voteview.py ../data/crawl/voteview/
	$(PYTHON) load/voteview.py
	touch $@

.earmarks: load/earmarks.py .govtrack .voteview
	$(PYTHON) load/earmarks.py
	touch $@

.bills: .govtrack load/bills.py
	$(PYTHON) load/bills.py

.interests: load/maplight.py .bills
	$(PYTHON) load/maplight.py
	touch $@

.votesmart: .json load/votesmart.py $(CRAWL)/votesmart/bios.json $(CRAWL)/votesmart/candidates.json
	$(PYTHON) load/votesmart.py
	touch $@

.photos: .json .votesmart load/photos.py
	$(PYTHON) load/photos.py
	touch $@

.fec: .json
	$(PYTHON) load/fec.py
	touch $@

.zip4: $(LOAD)/zip4.tsv .json
	# NOTE: leaving the index on while loading the data is unbearably slow. If
	#       the schema changes this makefile will need to be updated too.
	#       Also, our districts table is missing MH-00 (Marshall Islands) and
	#       thus the foreign key must be removed.
	echo "ALTER TABLE zip4 DROP CONSTRAINT zip4_pkey; \
ALTER TABLE zip4 DROP CONSTRAINT zip4_district_id_fkey; \
COPY zip4 (zip,plus4,district_id) FROM '${PWD}/../data/load/zip4.tsv'; \
ALTER TABLE zip4 ADD FOREIGN KEY (district_id) REFERENCES district; \
ALTER TABLE zip4 ADD PRIMARY KEY (zip,plus4); \
		" | $(DBCLIENT) $(WATCHDOG_TABLE)
	touch $@

.soi: .schema .json
	$(PYTHON) load/soi.py
	touch $@

$(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv: load/census.py parse/census.py
	python load/census.py

.census: $(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv .schema .json
	$(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_meta.tsv 
	$(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_data.tsv 
	touch $@<commit_msg>fix makefile bug<commit_after>ifeq ($(DATABASE_ENGINE),mysql)
    DBCLIENT=mysql
else ifeq ($(DATABASE_ENGINE),sqlite)
    DBCLIENT=sqlite3
else
    DBCLIENT=psql
endif

ifeq ($(WATCHDOG_TABLE),)
    WATCHDOG_TABLE=watchdog_dev
endif

export PYTHONPATH := ../:.:$(PWD)/../vendor:$(PYTHONPATH)

PYTHON=python

CRAWL=../data/crawl
PARSE=../data/parse
LOAD=../data/load

TSVS=$(LOAD)/bill.tsv \
	 $(LOAD)/vote.tsv \
	 $(LOAD)/zip4.tsv \
	 $(LOAD)/census_meta.tsv \
	 $(LOAD)/census_data.tsv

NJS=$(PARSE)/almanac.njs \
    $(PARSE)/earmarks.njs \
    $(PARSE)/fec.njs \
    $(PARSE)/govtrack.njs \
    $(PARSE)/soi.njs \
    $(PARSE)/voteview.njs

LOADERS = .schema .manual \
          .almanac .centers .voteview .govtrack \
          .bills .interests .votesmart .photos .fec .census .soi .earmarks

all: database
clean:
	rm $(LOADERS)

cleanload:
	rm $(TSVS)

cleanparse:
	rm $(NJS)

database: $(LOADERS)

.manual: .schema load/manual.py load/manual/*.json
	python load/manual.py
	touch $@

.schema: ../schema.py
	python -c "import schema; schema.sql.recreate(); schema.sql.grantall('watchdog')"
	touch $@

# not currently using .shapes
.shapes: .manual load/shapes.py parse/shapes.py
	$(PYTHON) load/shapes.py
	touch $@

.almanac: .manual load/almanac.py parse/almanac.py
	$(PYTHON) load/almanac.py
	touch $@

.centers: .manual load/govtrack_gis.py ../data/crawl/govtrack/gis/gmapdata/*
	$(PYTHON) load/govtrack_gis.py
	touch $@

.govtrack: load/govtrack.py parse/govtrack.py ../data/crawl/govtrack/us/110/repstats/*xml .manual
	$(PYTHON) load/govtrack.py
	touch $@

.voteview: .manual load/voteview.py parse/voteview.py ../data/crawl/voteview/
	$(PYTHON) load/voteview.py
	touch $@

.earmarks: load/earmarks.py .govtrack .voteview
	$(PYTHON) load/earmarks.py
	touch $@

.bills: .govtrack load/bills.py
	$(PYTHON) load/bills.py

.interests: load/maplight.py .bills
	$(PYTHON) load/maplight.py
	touch $@

.votesmart: .manual load/votesmart.py $(CRAWL)/votesmart/bios.json $(CRAWL)/votesmart/candidates.json
	$(PYTHON) load/votesmart.py
	touch $@

.photos: .manual .votesmart load/photos.py
	$(PYTHON) load/photos.py
	touch $@

.fec: .manual
	$(PYTHON) load/fec.py
	touch $@

.zip4: $(LOAD)/zip4.tsv .manual
	# NOTE: leaving the index on while loading the data is unbearably slow. If
	#       the schema changes this makefile will need to be updated too.
	#       Also, our districts table is missing MH-00 (Marshall Islands) and
	#       thus the foreign key must be removed.
	echo "ALTER TABLE zip4 DROP CONSTRAINT zip4_pkey; \
ALTER TABLE zip4 DROP CONSTRAINT zip4_district_id_fkey; \
COPY zip4 (zip,plus4,district_id) FROM '${PWD}/../data/load/zip4.tsv'; \
ALTER TABLE zip4 ADD FOREIGN KEY (district_id) REFERENCES district; \
ALTER TABLE zip4 ADD PRIMARY KEY (zip,plus4); \
		" | $(DBCLIENT) $(WATCHDOG_TABLE)
	touch $@

.soi: .schema .manual
	$(PYTHON) load/soi.py
	touch $@

$(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv: load/census.py parse/census.py
	python load/census.py

.census: $(LOAD)/census_meta.tsv $(LOAD)/census_data.tsv .schema .manual
	$(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_meta.tsv 
	$(DBCLIENT) $(WATCHDOG_TABLE) < $(LOAD)/census_data.tsv 
	touch $@
{'fraction_non_alphanumeric': 0.15870011042751223, 'fraction_numerical': 0.005994636377977599, 'mean_word_length': 3.577617328519856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2829905', 'n_tokens_mistral': 2702, 'n_tokens_neox': 2583, 'n_words': 522}