<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import mako
import json
import os

from girder import constants, events
from girder.utility.model_importer import ModelImporter
from girder.utility.webroot import Webroot

from girder.api.rest import Resource, loadmodel, RestException
from girder.api.describe import Description
from girder.api import access


class GeoParserJobs(Resource):
    def __init__(self):
        self.resourceName = 'geoparser_jobs'
        self.route('GET', ("extract_text",), self.extractText)
        self.route('GET', ("find_location",), self.findLocation)
        self.route('GET', ("find_lat_lon",), self.findLatlon)

    @access.public
    def extractText(self, params):
        ## Run Tika text extraction
        return {'data': 'some text'}
    extractText.description = (
        Description('Extract text')
    )

    @access.public
    def findLocation(self, params):
        # Run Geograpy location name finder from text
        return {'data': 'some location'}
    findLocation.description = (
        Description('Find location name')
    )

    @access.public
    def findLatlon(self, params):
        # Run Geopy latitude and longitude finder from location name
        return {'data': 'some lat lon'}
    findLatlon.description = (
        Description('Find latitude and longitude')
    )

<commit_msg>#24 adding three jobs detail functionality and Solr functions<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import mako
import json
import os

from girder import constants, events
from girder.utility.model_importer import ModelImporter
from girder.utility.webroot import Webroot

from girder.api.rest import Resource, loadmodel, RestException
from girder.api.describe import Description
from girder.api import access


from tika import parser
from geograpy import extraction
from geopy.geocoders import Nominatim

geolocator = Nominatim()

class GeoParserJobs(Resource):
    def __init__(self):
        self.resourceName = 'geoparser_jobs'
        self.route('GET', ("extract_text",), self.extractText)
        self.route('GET', ("find_location",), self.findLocation)
        self.route('GET', ("find_lat_lon",), self.findLatlon)


    def SolrIndexText(text):
        pass


    def SolrQueryText(file_name):
        pass


    def SolrIndexLocationName(places):
        pass


    def SolrQueryLocationName(file_name):
        pass


    def SolrIndexLatLon(points):
        pass


    @access.public
    def extractText(self, file_name):
        '''
        Using Tika to extract text from given file
        and return the text content.
        '''
        parsed = parser.from_file(file_name)
        SolrIndexText(parsed["content"])
        return {'data': 'Text Extracted'}
    extractText.description = (
        Description('Extract text')
    )


    @access.public
    def findLocation(self, file_name):
        '''
        Find location name from extracted text using Geograpy.
        '''
        text_content = SolrQueryText(file_name)
        e = extraction.Extractor(text=text_content)
        e.find_entities()
        SolrIndexLocationName(e.places)
        return {'data': 'Location name Found.'}
    findLocation.description = (
        Description('Find location name')
    )


    @access.public
    def findLatlon(self, file_name):
        '''
        Find latitude and longitude from location name using GeoPy.
        '''
        location_names = SolrQueryLocationName(file_name)
        points = []
        for location in location_names:
            try:
                geolocation = geolocator.geocode(location)
                points.append([geolocation.latitude, geolocation.longitude,location])
            except:
                pass
        SolrIndexLatLon(points)
        return {'data': 'Latitude and longitude Found.'}
    findLatlon.description = (
        Description('Find latitude and longitude')
    )


{'fraction_non_alphanumeric': 0.09614894159653149, 'fraction_numerical': 0.00102014792144861, 'mean_word_length': 2.837573385518591, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1062970', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1083, 'n_words': 291}