<commit_before># -*- coding: utf-8 -*-


import logging
from itertools import product, combinations

from py2neo import authenticate, Graph, Node, Relationship
from scrapy import signals
from scrapy.exporters import JsonLinesItemExporter, PprintItemExporter

from .latex import utils as latexutils


class EquationscraperPipeline(object):
    def __init__(self):
        self.jsl_exporter = None
        self.pprnt_exporter = None
        self.files = {}

        authenticate('localhost:7474', 'neo4j', 'big-theta-team')
        self.graph = Graph('localhost:7474/db/data')

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        file_pprnt = open('%s_pprint-items0' % spider.name, 'w+b', )
        file_jsl = open('%s_json-items0' % spider.name, 'w+b', )

        self.jsl_exporter = JsonLinesItemExporter(file_jsl)
        self.pprnt_exporter = PprintItemExporter(file_pprnt)

        self.files[spider] = [file_pprnt, file_jsl]
        self.pprnt_exporter.indent = 2
        self.pprnt_exporter.start_exporting()
        self.jsl_exporter.start_exporting()

    def spider_closed(self, spider):
        self.pprnt_exporter.finish_exporting()
        self.jsl_exporter.finish_exporting()

        for f in self.files[spider]:
            f.close()

    def process_item(self, item, spider):
        self.pprnt_exporter.export_item(item)
        # self.jsl_exporter.export_item(item)

        node_label = 'EQUATION'
        link_relation = 'LINKS_TO'
        page_relation = 'SAME_PAGE_AS'

        item_array = [item['last_item'].copy(), item.copy()]

        for idx, elem in enumerate(item_array):

            for expr_x, expr_y in combinations(elem['maths'], 2):

                expr_x = latexutils.strip_styles(expr_x)
                expr_y = latexutils.strip_styles(expr_y)

                logging.log(logging.INFO, "JPCLOG SAMELINK:" + str((expr_x, expr_y)))

                if (latexutils.contains_equality_command(expr_x) and
                        latexutils.contains_equality_command(expr_y)):
                    # we are not interested in math expressions without
                    # some form of equality. The expression sin(x)/x is meaningless
                    # but the expression lim_{x->0} sin(x)/x = 1 contains meaningful
                    # information i.e. that the LHS and RHS are related in some way--
                    # in this case they are equal.

                    node_pair = []
                    for latex_item in [expr_x, expr_y]:
                        node_pair.append(Node(node_label,
                                              equation=latex_item,
                                              title=item_array[idx]['title'],
                                              url=item_array[idx]['url'],<commit_msg>added setting for json-exporting items<commit_after># -*- coding: utf-8 -*-


import logging
from itertools import product, combinations

from py2neo import authenticate, Graph, Node, Relationship
from scrapy import signals
from scrapy.exporters import JsonLinesItemExporter, PprintItemExporter

from .latex import utils as latexutils



class EquationscraperPipeline(object):
    def __init__(self):
        self.jsl_exporter = None
        self.pprnt_exporter = None
        self.files = {}

        authenticate('localhost:7474', 'neo4j', 'big-theta-team')
        self.graph = Graph('localhost:7474/db/data')

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        file_pprnt = open('%s_pprint-items0' % spider.name, 'w+b', )
        file_jsl = open('%s_json-items0' % spider.name, 'w+b', )

        self.jsl_exporter = JsonLinesItemExporter(file_jsl)
        self.pprnt_exporter = PprintItemExporter(file_pprnt)

        self.files[spider] = [file_pprnt, file_jsl]
        self.pprnt_exporter.indent = 2
        self.pprnt_exporter.start_exporting()
        self.jsl_exporter.start_exporting()

    def spider_closed(self, spider):
        self.pprnt_exporter.finish_exporting()
        self.jsl_exporter.finish_exporting()

        for f in self.files[spider]:
            f.close()

    def process_item(self, item, spider):

        self.pprnt_exporter.export_item(item)

        if spider.settings.getbool("EXPORT_JSON"):
            self.jsl_exporter.export_item(item)

        node_label = 'EQUATION'
        link_relation = 'LINKS_TO'
        page_relation = 'SAME_PAGE_AS'

        item_array = [item['last_item'].copy(), item.copy()]

        for idx, elem in enumerate(item_array):

            for expr_x, expr_y in combinations(elem['maths'], 2):

                expr_x = latexutils.strip_styles(expr_x)
                expr_y = latexutils.strip_styles(expr_y)

                logging.log(logging.INFO, "JPCLOG SAMELINK:" + str((expr_x, expr_y)))

                if (latexutils.contains_equality_command(expr_x) and
                        latexutils.contains_equality_command(expr_y)):
                    # we are not interested in math expressions without
                    # some form of equality. The expression sin(x)/x is meaningless
                    # but the expression lim_{x->0} sin(x)/x = 1 contains meaningful
                    # information i.e. that the LHS and RHS are related in some way--
                    # in this case they are equal.

                    node_pair = []
                    for latex_item in [expr_x, expr_y]:
                        node_pair.append(Node(node_label,
                                              equation=latex_item,
                                              title=item_array[idx]['title'],
                                              url=item_array[idx]['url'],
{'fraction_non_alphanumeric': 0.09449975518198139, 'fraction_numerical': 0.00554920842173984, 'mean_word_length': 2.168562564632885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5508859', 'n_tokens_mistral': 1880, 'n_tokens_neox': 1773, 'n_words': 369}