<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
import csv
import json
import requests
import scraperwiki
import config as Config
from hdx_format import item
from termcolor import colored as color
from store_records import StoreRecords

dir = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]


def QueryWFP(endpoint, parameters_dict, verbose = False, make_json = False, make_csv = False, store_db = True, log = 'scraperwiki'):
  '''Query WFP's VAM API.'''

  u = Config.BuildQueryString(endpoint, parameters_dict)

  if verbose:
    print u
  
  r = requests.get(u)
  if r.status_code != 200:
    print "Query returned error code: %s" % r

  else:
    data = r.json()
    info = [ key + '=' + parameters_dict[key] for key in parameters_dict.keys() ]
    info_string = ", ".join(info)

    if len(data) == 0:
      if log == 'scraperwiki':
        print "Data not found, %s" % (info_string)
      else:
        print "%s %s Data not found for %s" % (item('bullet_red'), color(endpoint, 'yellow'), info_string)

    if len(data) > 0:
      if log == 'scraperwiki':
        "Data found, %s" % (info_string)

      else:
        print "%s %s Data found for %s" % (item('bullet_green'), color(endpoint, 'yellow'), info_string)

      if make_json:
        j_path = os.path.join(dir, 'data/') + endpoint + '_data.json'
        with open(j_path, 'w') as outfile:
          json.dump(data, outfile)

      if make_csv:
        c_path = os.path.join(dir, 'data/') + endpoint + '_data_.csv'

        f = csv.writer(open(c_path, "wb+"))
        f.writerow(data[0].keys())

        for row in data:
          f.writerow([ row[key] if isinstance(row[key], dict) is False else row[key].values()[0] for key in row.keys() ])

      if store_db:
        for row in data:
          record = [{ key:row[key] if isinstance(row[key], dict) is False else row[key].values()[0] for key in row.keys() }]
          StoreRecords(record, endpoint)


# def CreateUniqueCode():
  # '''Creating unique codes for locations.'''

def Main():
  '''Wrapper.'''

  l = Config.LoadListOfLocations()
  for row in l:
    parameters_dict = Config.RecurseDisagreggation(row=row)

    # for testing
    # parameters_dict = {'adm0': '239', 'indTypeID': '1'}

    # iterating over the type.
    type_ids = ['1', '2']
    csi_types = ['r', 'cs']
    for type in type_ids:
      parameters_dict["indTypeID"] = type

      # query Income and FSC
      QueryWFP("FCS", parameters_dict = parameters_dict)
      QueryWFP("Income", parameters_dict = parameters_dict)

      # add extra parameter for CSI.
      for csi_type in csi_types:
        parameters_dict["type"] = csi_type
        QueryWFP("CSI", parameters_dict = parameters_dict)

      parameters_dict.pop("type", None)  # deleting the entry



if __name__ == '__main__':

  try:
      Main()
      print "SW Status: Everything seems to be just fine."
      scraperwiki.status('ok')

  except Exception as e:
      print e
      scraperwiki.status('error', 'Error collecting data.')
      os.system("echo https://ds-ec2.scraperwiki.com/deudwxf/bfbenh2isykhn56/http/log.txt | mail -s 'WFP APIs: Failed collecting data.' <EMAIL>")<commit_msg>adding error handler to the main query function<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import sys
import csv
import json
import requests
import scraperwiki
import config as Config
from hdx_format import item
from termcolor import colored as color
from store_records import StoreRecords

dir = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]


def QueryWFP(endpoint, parameters_dict, verbose = False, make_json = False, make_csv = False, store_db = True, log = 'scraperwiki'):
  '''Query WFP's VAM API.'''

  u = Config.BuildQueryString(endpoint, parameters_dict)

  if verbose:
    print u
  
  r = requests.get(u)
  if r.status_code != 200:
    print "Query returned error code: %s" % r

  else:
    data = r.json()
    info = [ key + '=' + parameters_dict[key] for key in parameters_dict.keys() ]
    info_string = ", ".join(info)

    if len(data) == 0:
      if log == 'scraperwiki':
        print "Data not found, %s" % (info_string)
      else:
        print "%s %s Data not found for %s" % (item('bullet_red'), color(endpoint, 'yellow'), info_string)

    if len(data) > 0:
      if log == 'scraperwiki':
        "Data found, %s" % (info_string)

      else:
        print "%s %s Data found for %s" % (item('bullet_green'), color(endpoint, 'yellow'), info_string)

      if make_json:
        j_path = os.path.join(dir, 'data/') + endpoint + '_data.json'
        with open(j_path, 'w') as outfile:
          json.dump(data, outfile)

      if make_csv:
        c_path = os.path.join(dir, 'data/') + endpoint + '_data_.csv'

        f = csv.writer(open(c_path, "wb+"))
        f.writerow(data[0].keys())

        for row in data:
          f.writerow([ row[key] if isinstance(row[key], dict) is False else row[key].values()[0] for key in row.keys() ])

      if store_db:
        for row in data:
          record = [{ key:row[key] if isinstance(row[key], dict) is False else row[key].values()[0] for key in row.keys() }]
          StoreRecords(record, endpoint)


# def CreateUniqueCode():
  # '''Creating unique codes for locations.'''

def Main():
  '''Wrapper.'''

  l = Config.LoadListOfLocations()
  for row in l:
    parameters_dict = Config.RecurseDisagreggation(row=row)

    # for testing
    # parameters_dict = {'adm0': '239', 'indTypeID': '1'}

    # iterating over the type.
    type_ids = ['1', '2']
    csi_types = ['r', 'cs']
    for type in type_ids:
      parameters_dict["indTypeID"] = type

      try:

        # query Income and FSC
        QueryWFP("FCS", parameters_dict = parameters_dict)
        QueryWFP("Income", parameters_dict = parameters_dict)

        # add extra parameter for CSI.
        for csi_type in csi_types:
          parameters_dict["type"] = csi_type
          QueryWFP("CSI", parameters_dict = parameters_dict)

        parameters_dict.pop("type", None)  # deleting the entry

      except Exception as e:
        print "%s Connection failed." % item('prompt_error')



if __name__ == '__main__':

  try:
      Main()
      print "SW Status: Everything seems to be just fine."
      scraperwiki.status('ok')

  except Exception as e:
      print e
      scraperwiki.status('error', 'Error collecting data.')
      os.system("echo https://ds-ec2.scraperwiki.com/deudwxf/bfbenh2isykhn56/http/log.txt | mail -s 'WFP APIs: Failed collecting data.' <EMAIL>")
{'fraction_non_alphanumeric': 0.13652346735973092, 'fraction_numerical': 0.006726800183458187, 'mean_word_length': 2.960048426150121, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5674968', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2127, 'n_words': 612}