<commit_before>import json
import urllib2

from bson import json_util
import cherrypy
from pandas import read_csv

from config.db import db
from lib.constants import BAMBOO_ID, SOURCE
from lib.utils import df_to_mongo, mongo_decode_keys, df_to_hexdigest
import models.collection

class Collections(object):

    def __init__(self):
        pass

    exposed = True

    def DELETE(self, id=None):
        '''
        Delete id 'id' from mongo
        '''
        if id:
            db().collections.remove({SOURCE: id})
            return 'deleted id: %s' % id

    def GET(self, id=None, format='json', query=None):
        '''
        Return data set for id 'id' in format 'format'.
        Execute query 'query' in mongo if passed.
        '''
        if query:
            try:
                query = json.loads(query, object_hook=json_util.object_hook)
            except ValueError, e:
                return e.message
        else:
            query = {}
        if id:
            query[BAMBOO_ID] = id
            json_str = mongo_decode_keys([
                r for r in db().collections.find(query)
            ])
            return json.dumps(json_str, default=json_util.default)

    def POST(self, url=None, data=None):
        '''
        Read data from URL 'url'.
        If URL is not provided and data is provided, read posted data 'data'.
        '''
        if url and 'http://' in url or 'https://' in url:
            f = urllib2.urlopen(url)
            df = read_csv(f, na_values=['n/a'])
            digest = df_to_hexdigest(df)
            num_rows_with_digest = db().collections.find(
                    {BAMBOO_ID: digest}).count()
            if not num_rows_with_digest:
                df = df_to_mongo(df)
                # add metadata to file
                for e in df:
                    e[BAMBOO_ID] = digest
                    e[SOURCE] = url
                # insert data into collections
                db().collections.insert(df)
            return json.dumps({'id': digest})<commit_msg>work on abstracting the data file import<commit_after>import json
import urllib2

from bson import json_util
import cherrypy
from pandas import read_csv

from config.db import db
from lib.constants import BAMBOO_ID, SOURCE
from lib.utils import df_to_mongo, mongo_decode_keys, df_to_hexdigest
import models.collection

class Collections(object):

    def __init__(self):
        pass

    exposed = True

    def DELETE(self, id=None):
        '''
        Delete id 'id' from mongo
        '''
        if id:
            db().collections.remove({SOURCE: id})
            return 'deleted id: %s' % id

    def GET(self, id=None, format='json', query=None):
        '''
        Return data set for id 'id' in format 'format'.
        Execute query 'query' in mongo if passed.
        '''
        if query:
            try:
                query = json.loads(query, object_hook=json_util.object_hook)
            except ValueError, e:
                return e.message
        else:
            query = {}
        if id:
            query[BAMBOO_ID] = id
            json_str = mongo_decode_keys([
                r for r in db().collections.find(query)
            ])
            return json.dumps(json_str, default=json_util.default)

    def POST(self, url=None, data=None):
        '''
        Read data from URL 'url'.
        If URL is not provided and data is provided, read posted data 'data'.
        '''
        def open_data_file(url):
            import re
            open_url = lambda url: urllib2.urlopen(url)
            protocols = {
                'http':  open_url,
                'https': open_url,
                'file':  lambda path: path,
            }
            regex = re.compile(
                '^(?P<url>(?P<protocol>%s):\/\/(?P<path>.+))$' \
                % '|'.join(protocols)
            )
            match = re.match(regex, url)
            if match:
                protocols[match.groupdict()['protocol']](url)
            return None
        f = open_data_file(url)
        if f:
            df = read_csv(f, na_values=['n/a'])
            digest = df_to_hexdigest(df)
            num_rows_with_digest = db().collections.find(
                    {BAMBOO_ID: digest}).count()
            if not num_rows_with_digest:
                df = df_to_mongo(df)
                # add metadata to file
                for e in df:
                    e[BAMBOO_ID] = digest
                    e[SOURCE] = url
                # insert data into collections
                db().collections.insert(df)
            return json.dumps({'id': digest})
{'fraction_non_alphanumeric': 0.09958776307224995, 'fraction_numerical': 0.0008678672163159037, 'mean_word_length': 1.6178307779670642, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7455958', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1322, 'n_words': 368}