<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals

import uuid
import tempfile

import pysolr
import requests


def send_task(name, *args):
    """
    Helper for sending a Celery task.

    ``name`` is the name of the task to send. If the name doesn't
    contain a ``.`` then it is prefixed with ``extractor.``.

    Any remaining arguments are passed to the task.
    """
    # Late import at call time because it requires a running app
    from ckan.lib.celery_app import celery
    if '.' not in name:
        name = 'extractor.' + name
    return celery.send_task(name, args, task_id=str(uuid.uuid4()))


def extract_metadata(document_url, solr_url):
    """
    Extract Metadata from content available at an URL.
    """
    # FIXME: Fulltext is returned as HTML
    with tempfile.NamedTemporaryFile() as f:
        print('Created temporary file {}'.format(f.name))
        r = requests.get(document_url, stream=True)
        for chunk in r.iter_content(chunk_size=1024):
            f.write(chunk)
        f.flush()
        f.seek(0)
        print('Finished download from {}'.format(document_url))
        print('Uploading to {} for metadata extraction'.format(solr_url))
        return pysolr.Solr(solr_url).extract(f)

<commit_msg>Extract fulltext as raw text instead of HTML.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function, unicode_literals

import uuid
import tempfile

import pysolr
import requests


def send_task(name, *args):
    """
    Helper for sending a Celery task.

    ``name`` is the name of the task to send. If the name doesn't
    contain a ``.`` then it is prefixed with ``extractor.``.

    Any remaining arguments are passed to the task.
    """
    # Late import at call time because it requires a running app
    from ckan.lib.celery_app import celery
    if '.' not in name:
        name = 'extractor.' + name
    return celery.send_task(name, args, task_id=str(uuid.uuid4()))


def extract_metadata(document_url, solr_url):
    """
    Extract Metadata from content available at an URL.
    """
    with tempfile.NamedTemporaryFile() as f:
        print('Created temporary file {}'.format(f.name))
        r = requests.get(document_url, stream=True)
        for chunk in r.iter_content(chunk_size=1024):
            f.write(chunk)
        f.flush()
        f.seek(0)
        print('Finished download from {}'.format(document_url))
        print('Uploading to {} for metadata extraction'.format(solr_url))
        return pysolr.Solr(solr_url).extract(f, extractFormat='text')


{'fraction_non_alphanumeric': 0.10848348348348348, 'fraction_numerical': 0.005255255255255256, 'mean_word_length': 3.0440060698027316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '204487', 'n_tokens_mistral': 827, 'n_tokens_neox': 801, 'n_words': 258}