<commit_before># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import requests
import json
import logging

from just.items import JustPublication
import logging

API_KEY = 'justitie-very-secret-key'
API_PUBLICATIONS = 'http://czl-api.code4.ro/api/publications/'

class JustPublicationsToApiPipeline(object):
    def process_item(self, item, spider):

        if type(item) != JustPublication:
            return item

        r = requests.post(API_PUBLICATIONS, json=dict(item), headers={'Authorization': 'Token %s' % (API_KEY,) } )

        api_log = logging.getLogger('api-log.txt')
        if r.status_code == 200 or r.status_code == '200':
            api_log.log(r.status_code, level=logging.INFO)
        else:
            api_log.log(r.status_code, level=logging.ERROR)
        api_log.log(r.content, level=logging.INFO)

        return item<commit_msg>log in the general scrapy log<commit_after># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import requests
import json
import logging

from just.items import JustPublication
import logging

API_KEY = 'justitie-very-secret-key'
API_PUBLICATIONS = 'http://czl-api.code4.ro/api/publications/'

class JustPublicationsToApiPipeline(object):
    def process_item(self, item, spider):

        if type(item) != JustPublication:
            return item

        r = requests.post(API_PUBLICATIONS, json=dict(item), headers={'Authorization': 'Token %s' % (API_KEY,) } )


        if r.status_code == 200 or r.status_code == '200':
            logging.log(msg=r.status_code, level=logging.INFO)
        else:
            logging.log(msg=r.status_code, level=logging.ERROR)
            logging.log(msg=r.content, level=logging.INFO)

        return item
{'fraction_non_alphanumeric': 0.11997991967871485, 'fraction_numerical': 0.008032128514056224, 'mean_word_length': 3.304535637149028, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4409397', 'n_tokens_mistral': 661, 'n_tokens_neox': 629, 'n_words': 144}