<commit_before>import urllib2
import urllib
import json
import pprint

import metautils

from dbsettings import settings

metautils.setsettings(settings)

#This is a one time operation to create organisations based on the originating portal in ODM DB
#Run prior to importing data

#Change these as appropriate
url = ''
apikey = ''

dict_cur = metautils.getDBCursor(settings, dictCursor = True)

#Create step. Failures are OK if the city already exists
dict_cur.execute("SELECT city_shortname FROM cities")

for rec in dict_cur.fetchall():
    dataset_dict = {}
    dataset_dict['name'] = rec['city_shortname']
    dataset_dict['id'] = rec['city_shortname']
    data_string = urllib.quote(json.dumps(dataset_dict))
    request = urllib2.Request(url +'/api/3/action/organization_create')
    request.add_header('Authorization', apikey)
    response = urllib2.urlopen(request, data_string)
    response_dict = json.loads(response.read())
    pprint.pprint(rec['city_shortname'] + ": " + str(response_dict['success']))

#Update step. We always want to be able to update.
keys = ['city_shortname', 'city_fullname']
extras = ['city_type', 'open_data_portal', 'api_get', 'api_post', 'api_link', 'api_post_content', 'latitude', 'longitude', 'polygon', 'contact_email', 'contact_person', 'url', 'ris_url']
query = 'SELECT '
keys.extend(extras)
for key in keys:
    query += key + ','
query = query[0:len(query)-1]
query = query + ' FROM cities'
dict_cur.execute(query)
for rec in dict_cur.fetchall():
    dataset_dict = {}
    dataset_dict['id'] = rec['city_shortname']
    dataset_dict['name'] = rec['city_shortname']
    dataset_dict['title'] = rec['city_fullname']
    dataset_dict['extras'] = []
    for key in extras:
        if rec[key] is not None:
            dataset_dict['extras'].append({'key': key, 'value': rec[key]})
    data_string = urllib.quote(json.dumps(dataset_dict))
    request = urllib2.Request(url +'/api/3/action/organization_create')
    request.add_header('Authorization', apikey)
    response = urllib2.urlopen(request, data_string)
    response_dict = json.loads(response.read())
    pprint.pprint(rec['city_shortname'] + ": " + str(response_dict['success']))
    assert(response_dict['success'] == True)
<commit_msg>Only add org fields that are not null. Add polygons.<commit_after>import urllib2
import urllib
import json
import pprint

import metautils

from dbsettings import settings

metautils.setsettings(settings)

#This is a one time operation to create organisations based on the originating portal in ODM DB
#Run prior to importing data

#Change these as appropriate
url = 'http://4192.168.127.12:5050'
apikey = '6798b059-4383-48db-900b-ed95f6a1effe'

dict_cur = metautils.getDBCursor(settings, dictCursor = True)

#Create step. Failures are OK if the city already exists
dict_cur.execute("SELECT city_shortname FROM cities")

for rec in dict_cur.fetchall():
    dataset_dict = {}
    dataset_dict['name'] = rec['city_shortname']
    dataset_dict['id'] = rec['city_shortname']
    data_string = urllib.quote(json.dumps(dataset_dict))
    request = urllib2.Request(url +'/api/3/action/organization_create')
    request.add_header('Authorization', apikey)
    response = urllib2.urlopen(request, data_string)
    response_dict = json.loads(response.read())
    pprint.pprint(rec['city_shortname'] + ": " + str(response_dict['success']))

#Update step. We always want to be able to update.
keys = ['city_shortname', 'city_fullname']
extras = ['city_type', 'open_data_portal', 'api_get', 'api_post', 'api_link', 'api_post_content', 'latitude', 'longitude', 'polygon', 'contact_email', 'contact_person', 'url', 'ris_url']
query = 'SELECT '
keys.extend(extras)
for key in keys:
    query += key + ','
query = query[0:len(query)-1]
query = query + ' FROM cities'
dict_cur.execute(query)
for rec in dict_cur.fetchall():
    dataset_dict = {}
    dataset_dict['id'] = rec['city_shortname']
    dataset_dict['name'] = rec['city_shortname']
    dataset_dict['title'] = rec['city_fullname']
    dataset_dict['extras'] = []
    for key in extras:
        if rec[key] is not None:
            if type(rec[key]) is dict:
                value = json.dumps(rec[key])
            else:
                value = rec[key]
            dataset_dict['extras'].append({'key': key, 'value': value})
    data_string = urllib.quote(json.dumps(dataset_dict))
    request = urllib2.Request(url +'/api/3/action/organization_create')
    request.add_header('Authorization', apikey)
    response = urllib2.urlopen(request, data_string)
    response_dict = json.loads(response.read())
    pprint.pprint(rec['city_shortname'] + ": " + str(response_dict['success']))
    assert(response_dict['success'] == True)

{'fraction_non_alphanumeric': 0.13763806287170774, 'fraction_numerical': 0.011469838572642312, 'mean_word_length': 4.680337756332931, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4413268', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1540, 'n_words': 356}