<commit_before>#! /bin/bash
#
# Copyright (C) 2014 by <NAME> (<NAME>
#
# This program is licensed to you under the terms of version 3 of the
# GNU Affero General Public License. This program is distributed WITHOUT
# ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
# AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
#

# do the committees in a simple query
/usr/bin/time psql -d us_geocoder -f ./geocode-committees.sql 

# need grouped updates for the transactions - takes too long
while [ `psql -q -d us_geocoder < count.sql` -gt 0 ]
do
  /usr/bin/time psql -d us_geocoder < batch-geocode-transactions.sql
  psql -q -d us_geocoder < count.sql
done<commit_msg>parallel jobs<commit_after>#! /bin/bash
#
# Copyright (C) 2014 by <NAME> (<NAME>
#
# This program is licensed to you under the terms of version 3 of the
# GNU Affero General Public License. This program is distributed WITHOUT
# ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
# MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
# AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
#

# do the committees in a simple query
/usr/bin/time psql -d us_geocoder -f ./geocode-committees.sql 

# need grouped updates for the transactions - takes too long
while [ `psql -q -d us_geocoder < count.sql` -gt 0 ]
do
  time psql -d us_geocoder < bgt0.sql &
  time psql -d us_geocoder < bgt1.sql &
  time psql -d us_geocoder < bgt2.sql &
  time psql -d us_geocoder < bgt3.sql &
  time psql -d us_geocoder < bgt4.sql &
  time psql -d us_geocoder < bgt5.sql &
  wait
  psql -q -d us_geocoder < count.sql
done
{'fraction_non_alphanumeric': 0.0951276102088167, 'fraction_numerical': 0.012761020881670533, 'mean_word_length': 3.5634920634920637, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '921130', 'n_tokens_mistral': 619, 'n_tokens_neox': 586, 'n_words': 224}