<commit_before>SHELL := bash

redo: redownload new-boundaries.osm.xml

redownload: rmosmdata boundaries.osm.xml townlands-no-geom.csv baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv

all: boundaries.osm.xml townlands-no-geom.csv baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv

rmosmdata:
	-rm boundaries.osm.xml new-boundaries.osm.xml
	-rm *-no-geom.csv*
	-rm ireland-and-northern-ireland.osm.pbf

clean:
	git clean -x -f

ireland-and-northern-ireland.osm.pbf:
	wget -O ireland-and-northern-ireland.osm.pbf http://planet.openstreetmap.ie/ireland-and-northern-ireland.osm.pbf

%-no-geom.csv.zip:
	wget http://www.townlands.ie/static/downloads/$@

%-no-geom.csv: %-no-geom.csv.zip
	-rm -f $@
	aunpack $<

boundaries.osm.xml: ireland-and-northern-ireland.osm.pbf
	osmosis --read-pbf ireland-and-northern-ireland.osm.pbf --tag-filter reject-way --tag-filter reject-node --tag-filter accept-relations admin_level=* outPipe.0=admin_level --read-pbf ireland-and-northern-ireland.osm.pbf --tag-filter reject-way --tag-filter reject-node --tag-filter accept-relations boundary=* outPipe.0=boundaries --merge inPipe.0=admin_level inPipe.1=boundaries --write-xml boundaries.osm.xml
	xmlstarlet c14n boundaries.osm.xml > boundaries2.osm.xml
	mv boundaries2.osm.xml boundaries.osm.xml

new-boundaries.osm.xml: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	mkdir -p ./output/`date -I`
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --townlands | tee >( lzma > ./output/`date -I`/output.lzma)
	xmlstarlet c14n new-boundaries.osm.xml > new-boundaries2.osm.xml
	mv new-boundaries2.osm.xml new-boundaries.osm.xml

bar-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --dry-run

cp-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --dry-run

td-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --townlands --dry-run

sample: clean new-boundaries.osm.xml boundaries.osm.xml
	tar -cf sample-data-`date -I`.tar boundaries.osm.xml new-boundaries.osm.xml
	lzma sample-data-`date -I`.tar

logainm-csvs.zip:
	wget -O logainm-csvs.zip http://www.technomancy.org/logainm/logainm-csvs.zip

logainm-csvs/logainm_names.csv: logainm-csvs.zip
	aunpack logainm-csvs.zip
	# Without this, the datetime in logainm_name.csvs will be old and hence
	# it'll always try to rename thistarget
	touch logainm-csvs/*

logainm.sqlite: logainm-csvs/logainm_names.csv
	-rm -f logainm.sqlite
	cat csv2sqlite.sql | sqlite3 logainm.sqlite

lint: boundaries.osm.xml
	python logainm_lint.py -i boundaries.osm.xml --dupe-logainm-ref
<commit_msg>don't download and extract the csv<commit_after>SHELL := bash

redo: redownload new-boundaries.osm.xml

redownload: rmosmdata boundaries.osm.xml townlands-no-geom.csv baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv

all: boundaries.osm.xml townlands-no-geom.csv baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv

rmosmdata:
	-rm boundaries.osm.xml new-boundaries.osm.xml
	-rm *-no-geom.csv*
	-rm ireland-and-northern-ireland.osm.pbf

clean:
	git clean -x -f

ireland-and-northern-ireland.osm.pbf:
	wget -O ireland-and-northern-ireland.osm.pbf http://planet.openstreetmap.ie/ireland-and-northern-ireland.osm.pbf

%-no-geom.csv.zip:
	wget http://www.townlands.ie/static/downloads/$@

%-no-geom.csv: %-no-geom.csv.zip
	-rm -f $@
	aunpack $<

boundaries.osm.xml: ireland-and-northern-ireland.osm.pbf
	osmosis --read-pbf ireland-and-northern-ireland.osm.pbf --tag-filter reject-way --tag-filter reject-node --tag-filter accept-relations admin_level=* outPipe.0=admin_level --read-pbf ireland-and-northern-ireland.osm.pbf --tag-filter reject-way --tag-filter reject-node --tag-filter accept-relations boundary=* outPipe.0=boundaries --merge inPipe.0=admin_level inPipe.1=boundaries --write-xml boundaries.osm.xml
	xmlstarlet c14n boundaries.osm.xml > boundaries2.osm.xml
	mv boundaries2.osm.xml boundaries.osm.xml

new-boundaries.osm.xml: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	mkdir -p ./output/`date -I`
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --townlands | tee >( lzma > ./output/`date -I`/output.lzma)
	xmlstarlet c14n new-boundaries.osm.xml > new-boundaries2.osm.xml
	mv new-boundaries2.osm.xml new-boundaries.osm.xml

bar-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --dry-run

cp-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --dry-run

td-dry-run: boundaries.osm.xml logainm.sqlite match.py townlands-no-geom.csv \
	baronies-no-geom.csv civil_parishes-no-geom.csv counties-no-geom.csv
	python match.py --verbose --input boundaries.osm.xml --output new-boundaries.osm.xml --baronies --civil-parishes --townlands --dry-run

sample: clean new-boundaries.osm.xml boundaries.osm.xml
	tar -cf sample-data-`date -I`.tar boundaries.osm.xml new-boundaries.osm.xml
	lzma sample-data-`date -I`.tar

lint: boundaries.osm.xml
	python logainm_lint.py -i boundaries.osm.xml --dupe-logainm-ref

{'fraction_non_alphanumeric': 0.14177663340843258, 'fraction_numerical': 0.0041841004184100415, 'mean_word_length': 3.817829457364341, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 74}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1927296', 'n_tokens_mistral': 2555, 'n_tokens_neox': 2474, 'n_words': 414}