<commit_before>#!/usr/bin/env python
import sys
import ckan    
import urllib2
from datetime import date, datetime
import os
from iatidataquality import models, db,  DATA_STORAGE_DIR
from iatidataquality.dqprocessing import add_hardcoded_result
import json
import daemon

import sys
import pprint
pp = pprint.PrettyPrinter(indent=2)

runtime = models.Runtime()
db.session.add(runtime)
db.session.commit()

download_queue = 'iati_download_queue'
import pika

def fixURL(url):
    # helper function to replace spaces with %20 (otherwise fails with some servers, e.g. US)
    url = url.replace(" ", "%20")
    return url

def create_package_group(group):
    pg = models.PackageGroup()
    pg.name = group
    pg.man_auto="auto"
    
    # Query CKAN
    url = 'http://iatiregistry.org/api'
    import ckanclient
    registry = ckanclient.CkanClient(base_location=url)
    startnow = False
    ckangroup = registry.group_entity_get(group)
    pg.title = ckangroup['title']
    pg.ckan_id = ckangroup['id']
    pg.revision_id = ckangroup['revision_id']
    pg.created_date = ckangroup['created']
    try:
        pg.state = ckangroup['state']
    except Exception, e:
        pass
    try:
        pg.license_id = ckangroup['extras']['publisher_license_id']
    except Exception, e:
        pass

    fields = [
        'publisher_iati_id', 'publisher_segmentation', 'publisher_type', 
        'publisher_ui', 'publisher_organization_type', 
        'publisher_frequency', 'publisher_thresholds', 'publisher_units', 
        'publisher_contact', 'publisher_agencies', 
        'publisher_field_exclusions', 'publisher_description', 
        'publisher_record_exclusions', 'publisher_timeliness', 
        'publisher_country', 'publisher_refs', 
        'publisher_constraints', 'publisher_data_quality'
        ]

    for field in fields:
        try:
            setattr(pg, field, ckangroup['extras'][field])
        except Exception, e:
            pass

    db.session.add(pg)
    db.session.commit()
    return pg

def metadata_to_db(pkg, success, update_package):
    if (update_package):
        package = models.Package.query.filter_by(package_ckan_id=pkg['id']).first()
    else:
        package = models.Package()
    package.man_auto = 'auto'
    package.source_url = pkg['resources'][0]['url']

    mapping = [
        ("package_ckan_id", "id"),
        ("package_name", "name"),
        ("package_title", "title"),
        ("package_license_id", "license_id"),
        ("package_license", "license"),
        ("package_metadata_created", "metadata_created"),
        ("package_metadata_modified", "metadata_modified"),
        ("package_revision_id", "revision_id")
        ]

    for attr, key in mapping:
        setattr(package, attr, pkg[key])

    try:
        # there is a group, so use that group ID, or create one
        group = pkg['groups'][0]
        try:
            pg = models.PackageGroup.query.filter_by(name=group).first()
            package.package_group = pg.id
        except Exception, e:
            pg = create_package_group(group)
            package.package_group = pg.id                
    except Exception, e:
        pass

    fields = [ 
        "activity_period-from", "activity_period-to",
        "activity_count", "country", "filetype", "verified" 
        ]
    for field in fields:
        try:
            field_name = "package_" + field.replace("-", "_")
            setattr(package, field_name, pkg["extras"][field])
        except Exception, e:
            pass

    db.session.add(package)
    db.session.commit()
    add_hardcoded_result(-2, runtime.id, package.id, success)
<commit_msg>factor out more common code<commit_after>#!/usr/bin/env python
import sys
import ckan    
import urllib2
from datetime import date, datetime
import os
from iatidataquality import models, db,  DATA_STORAGE_DIR
from iatidataquality.dqprocessing import add_hardcoded_result
import json
import daemon

import sys
import pprint
pp = pprint.PrettyPrinter(indent=2)

runtime = models.Runtime()
db.session.add(runtime)
db.session.commit()

download_queue = 'iati_download_queue'
import pika

def fixURL(url):
    # helper function to replace spaces with %20 (otherwise fails with some servers, e.g. US)
    url = url.replace(" ", "%20")
    return url

def create_package_group(group):
    pg = models.PackageGroup()
    pg.name = group
    pg.man_auto="auto"
    
    # Query CKAN
    url = 'http://iatiregistry.org/api'
    import ckanclient
    registry = ckanclient.CkanClient(base_location=url)
    startnow = False
    ckangroup = registry.group_entity_get(group)

    mapping = [
        ("title", "title"),
        ("ckan_id", "id"),
        ("revision_id", "revision_id"),
        ("created_date", "created"),
        ("state", "state")
        ]
    for attr, key in mapping:
        try:
            setattr(pg, attr, ckangroup[key])
        except Exception, e:
            pass

    try:
        pg.license_id = ckangroup['extras']['publisher_license_id']
    except Exception, e:
        pass

    fields = [
        'publisher_iati_id', 'publisher_segmentation', 'publisher_type', 
        'publisher_ui', 'publisher_organization_type', 
        'publisher_frequency', 'publisher_thresholds', 'publisher_units', 
        'publisher_contact', 'publisher_agencies', 
        'publisher_field_exclusions', 'publisher_description', 
        'publisher_record_exclusions', 'publisher_timeliness', 
        'publisher_country', 'publisher_refs', 
        'publisher_constraints', 'publisher_data_quality'
        ]

    for field in fields:
        try:
            setattr(pg, field, ckangroup['extras'][field])
        except Exception, e:
            pass

    db.session.add(pg)
    db.session.commit()
    return pg

def metadata_to_db(pkg, success, update_package):
    if (update_package):
        package = models.Package.query.filter_by(package_ckan_id=pkg['id']).first()
    else:
        package = models.Package()
    package.man_auto = 'auto'
    package.source_url = pkg['resources'][0]['url']

    mapping = [
        ("package_ckan_id", "id"),
        ("package_name", "name"),
        ("package_title", "title"),
        ("package_license_id", "license_id"),
        ("package_license", "license"),
        ("package_metadata_created", "metadata_created"),
        ("package_metadata_modified", "metadata_modified"),
        ("package_revision_id", "revision_id")
        ]

    for attr, key in mapping:
        setattr(package, attr, pkg[key])

    try:
        # there is a group, so use that group ID, or create one
        group = pkg['groups'][0]
        try:
            pg = models.PackageGroup.query.filter_by(name=group).first()
        except Exception, e:
            pg = create_package_group(group)
        finally:
            package.package_group = pg.id
    except Exception, e:
        pass

    fields = [ 
        "activity_period-from", "activity_period-to",
        "activity_count", "country", "filetype", "verified" 
        ]
    for field in fields:
        try:
            field_name = "package_" + field.replace("-", "_")
            setattr(package, field_name, pkg["extras"][field])
        except Exception, e:
            pass

    db.session.add(package)
    db.session.commit()
    add_hardcoded_result(-2, runtime.id, package.id, success)

{'fraction_non_alphanumeric': 0.1183528770111808, 'fraction_numerical': 0.0024543223343332426, 'mean_word_length': 2.806434872859367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2053636', 'n_tokens_mistral': 2307, 'n_tokens_neox': 2239, 'n_words': 490}