<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import requests
import dateutil.parser
from django.contrib.gis.geos import Polygon

from django.conf import settings

from apps.bigcz.models import Resource, ResourceLink, ResourceList, BBox
from apps.bigcz.utils import RequestTimedOutError


CATALOG_NAME = 'cinergi'
CATALOG_URL = 'http://192.168.127.12:8080/geoportal/opensearch'


def parse_date(value):
    return dateutil.parser.parse(value) if value else None


def parse_envelope(envelope):
    if 'coordinates' not in envelope:
        raise ValueError('Expected envelope to contain coordinates')

    coords = envelope['coordinates']

    northwest = coords[0]
    southeast = coords[1]

    xmin, ymin = northwest
    xmax, ymax = southeast

    return Polygon.from_bbox((xmin, ymin, xmax, ymax))


def parse_geom(source):
    """
    Return Polygon or MultiPolygon from source by combining all
    available envelope_geo objects.

    - envelope_geo may be absent, an object, or list of objects
    """
    envelope = source.get('envelope_geo')

    if not envelope:
        return None

    if isinstance(envelope, dict):
        envelope = [envelope]

    geom = parse_envelope(envelope[0])

    for item in envelope[1:]:
        geom |= parse_envelope(item)

    return geom


def parse_links(source):
    """
    Parse "links" from item source.

    - links_s may be absent, a string, or list of strings
    """
    result = []
    links = source.get('links_s', [])

    if isinstance(links, basestring):
        links = [links]

    for url in links:
        result.append(ResourceLink('details', url))

    return result


def parse_record(item):
    source = item['_source']
    geom = parse_geom(source)
    links = parse_links(source)
    return Resource(
        id=item['_id'],
        title=source['title'],
        description=source.get('description'),
        author=None,
        links=links,
        created_at=parse_date(source.get('sys_created_dt')),
        updated_at=parse_date(source.get('src_lastupdate_dt')),
        geom=geom)


def prepare_bbox(value):
    box = BBox(value)
    return '{},{},{},{}'.format(box.xmin, box.ymin, box.xmax, box.ymax)


def prepare_date(value):
    return value.strftime('%Y-%m-%d')


def prepare_time(from_date, to_date):
    value = prepare_date(from_date)
    if to_date:
        value = '{}/{}'.format(value, prepare_date(to_date))
    return value


def search(**kwargs):
    query = kwargs.get('query')
    to_date = kwargs.get('to_date')
    from_date = kwargs.get('from_date')
    bbox = kwargs.get('bbox')

    params = {
        'f': 'json',
        'sort': 'sys_modified_dt:desc',
    }

    if query:
        params.update({
            'q': query
        })
    if from_date:
        params.update({
            'time': prepare_time(from_date, to_date)
        })
    if bbox:
        params.update({
            'bbox': prepare_bbox(bbox)
        })

    try:
        response = requests.get(CATALOG_URL,
                                timeout=settings.BIGCZ_CLIENT_TIMEOUT,
                                params=params)
    except requests.Timeout:
        raise RequestTimedOutError()

    data = response.json()

    if 'hits' not in data:
        raise ValueError(data)

    results = data['hits']['hits']
    count = data['hits']['total']

    return ResourceList(<commit_msg>Add pagination support to CINERGI<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import requests
import dateutil.parser
from django.contrib.gis.geos import Polygon

from django.conf import settings

from apps.bigcz.models import Resource, ResourceLink, ResourceList, BBox
from apps.bigcz.utils import RequestTimedOutError


CATALOG_NAME = 'cinergi'
CATALOG_URL = 'http://192.168.127.12:8080/geoportal/opensearch'
PAGE_SIZE = settings.BIGCZ_CLIENT_PAGE_SIZE


def parse_date(value):
    return dateutil.parser.parse(value) if value else None


def parse_envelope(envelope):
    if 'coordinates' not in envelope:
        raise ValueError('Expected envelope to contain coordinates')

    coords = envelope['coordinates']

    northwest = coords[0]
    southeast = coords[1]

    xmin, ymin = northwest
    xmax, ymax = southeast

    return Polygon.from_bbox((xmin, ymin, xmax, ymax))


def parse_geom(source):
    """
    Return Polygon or MultiPolygon from source by combining all
    available envelope_geo objects.

    - envelope_geo may be absent, an object, or list of objects
    """
    envelope = source.get('envelope_geo')

    if not envelope:
        return None

    if isinstance(envelope, dict):
        envelope = [envelope]

    geom = parse_envelope(envelope[0])

    for item in envelope[1:]:
        geom |= parse_envelope(item)

    return geom


def parse_links(source):
    """
    Parse "links" from item source.

    - links_s may be absent, a string, or list of strings
    """
    result = []
    links = source.get('links_s', [])

    if isinstance(links, basestring):
        links = [links]

    for url in links:
        result.append(ResourceLink('details', url))

    return result


def parse_record(item):
    source = item['_source']
    geom = parse_geom(source)
    links = parse_links(source)
    return Resource(
        id=item['_id'],
        title=source['title'],
        description=source.get('description'),
        author=None,
        links=links,
        created_at=parse_date(source.get('sys_created_dt')),
        updated_at=parse_date(source.get('src_lastupdate_dt')),
        geom=geom)


def prepare_bbox(value):
    box = BBox(value)
    return '{},{},{},{}'.format(box.xmin, box.ymin, box.xmax, box.ymax)


def prepare_date(value):
    return value.strftime('%Y-%m-%d')


def prepare_time(from_date, to_date):
    value = prepare_date(from_date)
    if to_date:
        value = '{}/{}'.format(value, prepare_date(to_date))
    return value


def search(**kwargs):
    query = kwargs.get('query')
    to_date = kwargs.get('to_date')
    from_date = kwargs.get('from_date')
    bbox = kwargs.get('bbox')
    page = kwargs.get('page')

    params = {
        'f': 'json',
        'sort': 'sys_modified_dt:desc',
        'size': PAGE_SIZE,
    }

    if query:
        params.update({
            'q': query
        })
    if from_date:
        params.update({
            'time': prepare_time(from_date, to_date)
        })
    if bbox:
        params.update({
            'bbox': prepare_bbox(bbox)
        })
    if page:
        params.update({
            # page 1 is from 1, page 2 from 101, page 3 from 201, ...
            'from': PAGE_SIZE * (page - 1) + 1
        })

    try:
        response = requests.get(CATALOG_URL,
                                timeout=settings.BIGCZ_CLIENT_TIMEOUT,
                                params=params)
    except requests.Timeout:
        raise RequestTimedOutError()

    data = response.json()

    if 'hits' not in data:
        raise ValueError(data)

    results = data['hits']['hits']
    count = data['hits']['total']

    return ResourceList(
{'fraction_non_alphanumeric': 0.11630163570834488, 'fraction_numerical': 0.007208206265594677, 'mean_word_length': 2.701898409440739, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075109', 'n_tokens_mistral': 2343, 'n_tokens_neox': 2247, 'n_words': 544}