<commit_before>mongoose = require "mongoose"
async = require "async"
moment = require "moment"
moment.locale("zh-cn")
{StorySchema} = require "../model/schema"
crawler = require "./crawler"
Section = require "./section"
config = require "../config"
cache = require('express-redis-cache')
  host: config.redis.host, port: config.redis.port
Daily = {}

Daily.fetchBeforeDay = (date, cb) ->
  crawler.getData "http://news-at.zhihu.com/api/3/stories/before/" + date, (err, dayObj)->
    return cb err if err
    return cb null, dayObj

Daily.fetchStory = (story_id, cb)->
  Daily.getStory story_id, (err, storyObj)->
    if storyObj
      return cb new Error "StoryExist"
    else
      crawler.getData "http://news-at.zhihu.com/api/3/story/" + story_id, (err, storyObj)->
        return cb err if err
        if storyObj.theme_name
          return cb new Error "StoryNotFound"
        unless storyObj.image
          return cb new Error "ImangeNotFound"
        return cb null, storyObj

Daily.saveStory = (storyObj, date, cb)->
  crawler.upImage storyObj, (err, storyObj)->
    return cb err if err
    story = new StorySchema
      id: storyObj.id
      body: storyObj.body
      image_source: storyObj.image_source
      title: storyObj.title
      ga_prefix: storyObj.ga_prefix
      section_name: storyObj.section_name
      image: storyObj.image
      share_url: storyObj.share_url
      date: date
      publish_at: moment(date + storyObj.ga_prefix.substr(-2), "YYYYMMDDHH").format()<commit_msg>clean section cache<commit_after>mongoose = require "mongoose"
async = require "async"
moment = require "moment"
moment.locale("zh-cn")
{StorySchema} = require "../model/schema"
crawler = require "./crawler"
Section = require "./section"
config = require "../config"
cache = require('express-redis-cache')
  host: config.redis.host, port: config.redis.port
Daily = {}

Daily.fetchBeforeDay = (date, cb) ->
  crawler.getData "http://news-at.zhihu.com/api/3/stories/before/" + date, (err, dayObj)->
    return cb err if err
    return cb null, dayObj

Daily.fetchStory = (story_id, cb)->
  Daily.getStory story_id, (err, storyObj)->
    if storyObj
      return cb new Error "StoryExist"
    else
      crawler.getData "http://news-at.zhihu.com/api/3/story/" + story_id, (err, storyObj)->
        return cb err if err
        if storyObj.theme_name
          return cb new Error "StoryNotFound"
        unless storyObj.image
          return cb new Error "ImangeNotFound"
        return cb null, storyObj

Daily.saveStory = (storyObj, date, cb)->
  crawler.upImage storyObj, (err, storyObj)->
    return cb err if err
    if storyObj.section_name
      cache.del "/section", ->
      cache.del "/section/#{storyObj.section_name}", ->
    story = new StorySchema
      id: storyObj.id
      body: storyObj.body
      image_source: storyObj.image_source
      title: storyObj.title
      ga_prefix: storyObj.ga_prefix
      section_name: storyObj.section_name
      image: storyObj.image
      share_url: storyObj.share_url
      date: date
      publish_at: moment(date + storyObj.ga_prefix.substr(-2), "YYYYMMDDHH").format()
{'fraction_non_alphanumeric': 0.12171684817424727, 'fraction_numerical': 0.0019218449711723255, 'mean_word_length': 3.0348837209302326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67131', 'n_tokens_mistral': 1008, 'n_tokens_neox': 1001, 'n_words': 261}