<commit_before>__author__ = "<NAME>"


import digests 
import falcon
import json
import os
import rdflib


try:
    from config import config
except ImportError:
    config = {"debug": True,
              "cache": "Cache",
              "redis": {"host": "localhost",
		        "port": 6379,
		        "ttl": 604800},
              "rest_api": {"host": "localhost",
                           "port": 18150},
              # Blazegraph SPARQL Endpoint
              "triplestore": {"host": "localhost",
                              "port": 8080,
                              "path": "bigdata"},
              
    }

if config['cache'].startswith("TwemproxyCache"):
    from cache.twemproxy import TwemproxyCache
    CACHE = TwemproxyCache(**config)
elif config['cache'].startswith("ClusterCache"):
    from cache.cluster import ClusterCache
    CACHE = ClusterCache(**config)
else:
    from cache import Cache
    CACHE = Cache(**config)


rest = falcon.API()


# Hooks
def triple_key(req, resp, params):
    if len(params) < 1:
        params = req.params
    subj = params.get('s', None)
    pred = params.get('p', None)
    obj = params.get('o', None)
    triple_str, resp.body = None, None
    if subj and pred and obj:
        triple_str = CACHE.datastore.evalsha(
            CACHE.add_get_triple,
            3,
            subj,
            pred,
            obj)
        if triple_str and CACHE.datastore.exists(triple_str):
            triple_key = triple_str.decode()
            triple_digests = triple_key.split(":")
            resp.body = json.dumps(
                {"key": triple_str.decode(),
                 "subject_sha1": triple_digests[0],
                 "predicate_sha1": triple_digests[1],
                 "object_sha1": triple_digests[2]}
            )
        elif triple_str:
            resp.body = json.dumps(
                {"missing-triple-key": triple_str.decode()}
            )
        else:
            raise falcon.HTTPNotFound()

def get_triples(pattern):
    cursor = -1
    output = []
    iterations = 0
    while 1:
        iterations += 1
        if cursor == 0:
            break
        elif cursor < 0:
            cursor = 0
        cursor, resources = CACHE.datastore.scan(
	    cursor=cursor,
	    match=pattern,
	    count=1000)
        cursor = int(cursor)<commit_msg>Added code for subject search with Falcon API<commit_after>__author__ = "<NAME>"


import digests 
import falcon
import hashlib
import json
import os
import rdflib
import requests

try:
    from config import config
except ImportError:
    config = {"debug": True,
              "cache": "Cache",
              "redis": {"host": "localhost",
		        "port": 6379,
		        "ttl": 604800},
              "rest_api": {"host": "localhost",
                           "port": 18150},
              # Blazegraph SPARQL Endpoint
              "triplestore": {"host": "localhost",
                              "port": 8080,
                              "path": "bigdata"},
              
    }

if config['cache'].startswith("TwemproxyCache"):
    from cache.twemproxy import TwemproxyCache
    CACHE = TwemproxyCache(**config)
elif config['cache'].startswith("ClusterCache"):
    from cache.cluster import ClusterCache
    CACHE = ClusterCache(**config)
else:
    from cache import Cache
    CACHE = Cache(**config)


rest = falcon.API()


# Hooks
def triple_key(req, resp, params):
    if len(params) < 1:
        params = req.params
    subj = params.get('s', None)
    pred = params.get('p', None)
    obj = params.get('o', None)
    triple_str, resp.body = None, None
    if subj and pred and obj:
        triple_str = CACHE.datastore.evalsha(
            CACHE.add_get_triple,
            3,
            subj,
            pred,
            obj)
        if triple_str and CACHE.datastore.exists(triple_str):
            triple_key = triple_str.decode()
            triple_digests = triple_key.split(":")
            resp.body = json.dumps(
                {"key": triple_str.decode(),
                 "subject_sha1": triple_digests[0],
                 "predicate_sha1": triple_digests[1],
                 "object_sha1": triple_digests[2]}
            )
        elif triple_str:
            resp.body = json.dumps(
                {"missing-triple-key": triple_str.decode()}
            )
        else:
            raise falcon.HTTPNotFound()
    # Subject search
    if subj and not pred and not obj:
        pattern = "{}:*:*".format(hashlib.sha1(str(subj).encode()).hexdigest())
        output = {"subject": str(subj),
                  "predicate-objects": []}
        for triple_key in CACHE.datastore.keys(pattern):
            triples = triple_key.decode().split(":")
            output["predicate-objects"].append({"p": CACHE.datastore.get(triples[1]).decode(),
                                                "o": CACHE.datastore.get(triples[-1]).decode()})
        resp.body = json.dumps(output)

def get_triples(pattern):
    cursor = -1
    output = []
    iterations = 0
    while 1:
        iterations += 1
        if cursor == 0:
            break
        elif cursor < 0:
            cursor = 0
        cursor, resources = CACHE.datastore.scan(
	    cursor=cursor,
	    match=pattern,
	    count=1000)
        cursor = int(cursor)
{'fraction_non_alphanumeric': 0.11976501800265302, 'fraction_numerical': 0.014970627250331627, 'mean_word_length': 1.7940709370037056, 'pattern_counts': {'":': 43, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494255', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1541, 'n_words': 362}