<commit_before>#-*- encoding:utf-8 -*-

import re
import time
import psycopg2
import datetime

from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from scrapy.loader import ItemLoader
from scrapy.http import Request

from LjSpider.items import *
from LjSpider.Db.Mysql import *
from LjSpider.Exception import tryex

class EsfIrtSpider(CrawlSpider):
    name = 'lj_get_esf_irt'
    start_urls = []
    custom_settings = {
        'FEED_URI': '/usr/local/crawler/dxc/common/lj/data/lj_esf_irt_%s.csv' % datetime.date.today(),
        'JOBDIR': '/usr/local/crawler/dxc/common/lj/crawls/lj_esf_irt_%s' % datetime.date.today(),
        'LOG_FILE': '/usr/local/crawler/dxc/common/lj/logs/lj_esf_irt_%s.log' % datetime.date.today(),
        'DOWNLOADER_MIDDLEWARES':{
            'LjSpider.middlewares.ProxyMiddleware': 202,
            # 'LjSpider.middlewares.ProxyxxxMiddleware': 302,
        },
        'ITEM_PIPELINES':{
        #    'LjSpider.pipelines.InsertPostgresqlPipeline': 300,
            # 'LjSpider.pipelines.JsonHFPipeline': 301,
        }
    }

    def start_requests(self):
        id_esf_url = Postgresql().query('lj_residence', ['id', 'esf_url'])
        id_esf_url = Mysql().query_by_sql('''
                        select co.route,c.url
                        from t_web_lj_community co,t_web_lj_district d,t_web_lj_city c
                        where d.id=co.district_id and d.city_id=c.id
                    ''')
        for route_url in id_esf_url:
            yield Request(
                route_url['url'] + 'ershoufang/' + route_url['route'] + '/co32/',
                callback=self.get_esf_url,
                dont_filter=True
            )
        # return [Request(
        #     'https://nj.lianjia.com/ershoufang/hanzhongmendajie/co32/',
        #     callback=self.get_esf_url,
        #     dont_filter=True
        # )]

    def get_esf_url(self,response):
        into_it = Selector(response).xpath('/html/body/div[4]/div[1]/ul/li[1]/div[1]/div[4]/text()').extract_first()
        if not into_it:
            return
        fabu_time_gang = Selector(text=into_it).re(r'%s' % u'刚刚发布')
        fabu_time_tian = Selector(text=into_it).re(r'(\d+)%s' % u'天以前发布')
        if not fabu_time_gang and not fabu_time_tian:
            return
        if fabu_time_tian != [] and int(fabu_time_tian[0]) > 1:
            return

        esf_url = Selector(response).xpath('/html/body/div[4]/div[1]/ul/li/a/@href').extract()
        for url in esf_url:<commit_msg>final_to_csv<commit_after>#-*- encoding:utf-8 -*-

import re
import time
import psycopg2
import datetime

from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from scrapy.loader import ItemLoader
from scrapy.http import Request

from LjSpider.items import *
from LjSpider.Db.Mysql import *
from LjSpider.Exception import tryex

class EsfIrtSpider(CrawlSpider):
    name = 'lj_get_esf_irt'
    start_urls = []
    custom_settings = {
        'FEED_URI': '/usr/local/crawler/dxc/common/lj/data/lj_esf_irt_%s.csv' % datetime.date.today(),
        'JOBDIR': '/usr/local/crawler/dxc/common/lj/crawls/lj_esf_irt_%s' % datetime.date.today(),
        'LOG_FILE': '/usr/local/crawler/dxc/common/lj/logs/lj_esf_irt_%s.log' % datetime.date.today(),
        'DOWNLOADER_MIDDLEWARES':{
            'LjSpider.middlewares.ProxyMiddleware': 202,
            # 'LjSpider.middlewares.ProxyxxxMiddleware': 302,
        },
        'ITEM_PIPELINES':{
        #    'LjSpider.pipelines.InsertPostgresqlPipeline': 300,
            # 'LjSpider.pipelines.JsonHFPipeline': 301,
        }
    }

    def start_requests(self):
        id_esf_url = Mysql().query_by_sql('''
                        select co.route,c.url
                        from t_web_lj_community co,t_web_lj_district d,t_web_lj_city c
                        where d.id=co.district_id and d.city_id=c.id
                    ''')
        for route_url in id_esf_url:
            yield Request(
                route_url['url'] + 'ershoufang/' + route_url['route'] + '/co32/',
                callback=self.get_esf_url,
                dont_filter=True
            )
        # return [Request(
        #     'https://nj.lianjia.com/ershoufang/hanzhongmendajie/co32/',
        #     callback=self.get_esf_url,
        #     dont_filter=True
        # )]

    def get_esf_url(self,response):
        into_it = Selector(response).xpath('/html/body/div[4]/div[1]/ul/li[1]/div[1]/div[4]/text()').extract_first()
        if not into_it:
            return
        fabu_time_gang = Selector(text=into_it).re(r'%s' % u'刚刚发布')
        fabu_time_tian = Selector(text=into_it).re(r'(\d+)%s' % u'天以前发布')
        if not fabu_time_gang and not fabu_time_tian:
            return
        if fabu_time_tian != [] and int(fabu_time_tian[0]) > 1:
            return

        esf_url = Selector(response).xpath('/html/body/div[4]/div[1]/ul/li/a/@href').extract()
        for url in esf_url:
{'fraction_non_alphanumeric': 0.1302588996763754, 'fraction_numerical': 0.010922330097087379, 'mean_word_length': 2.370824812542604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7652889', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1787, 'n_words': 255}