<commit_before>/**
* Created by qing on 17-10-1.
*/
const axios = require('axios');
const RSS = require('rss');
const Queue = require('np-queue');
const logger = require('./logger');
const cache = require('./cache');
const utils = require('./utils');

// reuse SSL connection
const https = require('https');
const httpsAgent = new https.Agent({ keepAlive: true });

const axiosInstance = axios.create({
  timeout: 3000,
  httpsAgent
});

axiosInstance.interceptors.request.use(config => {
  logger.debug(`axios ${config.method} ${config.url}`);
  return config;
});

exports.fetchRSS = async function (uid, options) {
  if (!options) options = {};
  // 大图显示
  if (options.largePic === undefined) {
    options.largePic = true;
  }
  // TTL
  if (options.ttl === undefined) {
    options.ttl = 15;
  }
  // 表情图标
  if (options.emoji === undefined) {
    options.emoji = false;
  }

  // start
  const {
    userNotExist,
    requestSuccess,
    weiboData } = await getWeiboData(uid);

  let resultXML;
  if (weiboData) {
    // metadata
    var feed = new RSS({
      site_url: "https://weibo.com/" + uid,
      title: weiboData.user.screen_name + '的微博',
      description: weiboData.user.description,
      generator: 'https://github.com/zgq354/weibo-rss',
      ttl: options.ttl
    });
    // content
    weiboData.statuses.forEach(function (detail) {
      if (!detail) return;
      feed.item({
        title: detail.status_title || (detail.text ? detail.text.replace(/<[^>]+>/g, '').replace(/[\n]/g, '').substr(0, 25) : null),
        description: utils.formatStatus(detail, options.largePic, options.emoji),
        url: 'https://weibo.com/' + uid + '/' + detail.bid,
        guid: 'https://weibo.com/' + uid + '/' + detail.bid,
        date: new Date(detail.created_at)
      });
    });
    resultXML = feed.xml();
  }

  return {
    userNotExist,
    requestSuccess,
    resultXML
  };
};

// 通过用户的个性域名获取UID
exports.getUIDByDomain = function (domain) {
  // 利用手机版的跳转获取
  return axiosInstance.get(`https://m.weibo.cn/${domain}?&jumpfrom=weibocom`, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A356 Safari/604.1'
    },
    maxRedirects: 0,
  }).then(res => {
    // fake 404
    return Promise.reject(new Error('User Not Exist'));
  }).catch(err => {
    if (err.response) {
      if (err.response.status === 302) {
        const uid = err.response.headers.location.split("/u/")[1];
        return {
          uid
        };
      } else if (err.response.status === 404) {
        return {
          notFound: true,
        };
      }
    }
    return Promise.reject(err);
  });
};

/**
 * 获取最近发的微博
 */
async function getWeiboData(uid) {
  let resultList = [];
  let tempResultObject = await getByIndexAPI(uid);
  resultList.push(tempResultObject);
  if (!tempResultObject.requestSuccess || tempResultObject.userNotExist) {
    tempResultObject = await getByProfileInfo(uid);
    resultList.push(tempResultObject);
  }

  // result
  let weiboData;
  let userNotExist = resultList.every(item => item.userNotExist);
  let requestSuccess = resultList.some(item => item.requestSuccess);
  if (!userNotExist) {
    weiboData = await handleLongText(tempResultObject.weiboData);
  }

  return {
    userNotExist,
    requestSuccess,
    weiboData
  };
};

/**
 * 出错时自动暂停
 */
const avaliableMethod = {
  methods: {
    indexAPI: {
      enable: true,
      lastUpdatedTime: Date.now(),
      retryDelay: 600000,
    },
    profileInfo: {
      enable: true,
      lastUpdatedTime: Date.now(),<commit_msg>feat: add backup cache<commit_after>/**
* Created by qing on 17-10-1.
*/
const axios = require('axios');
const RSS = require('rss');
const Queue = require('np-queue');
const logger = require('./logger');
const cache = require('./cache');
const utils = require('./utils');

// reuse SSL connection
const https = require('https');
const httpsAgent = new https.Agent({ keepAlive: true });

const axiosInstance = axios.create({
  timeout: 3000,
  httpsAgent
});

axiosInstance.interceptors.request.use(config => {
  logger.debug(`axios ${config.method} ${config.url}`);
  return config;
});

exports.fetchRSS = async function (uid, options) {
  if (!options) options = {};
  // 大图显示
  if (options.largePic === undefined) {
    options.largePic = true;
  }
  // TTL
  if (options.ttl === undefined) {
    options.ttl = 15;
  }
  // 表情图标
  if (options.emoji === undefined) {
    options.emoji = false;
  }

  // start
  const {
    userNotExist,
    requestSuccess,
    weiboData } = await getWeiboData(uid);

  let resultXML;
  if (weiboData) {
    // metadata
    var feed = new RSS({
      site_url: "https://weibo.com/" + uid,
      title: weiboData.user.screen_name + '的微博',
      description: weiboData.user.description,
      generator: 'https://github.com/zgq354/weibo-rss',
      ttl: options.ttl
    });
    // content
    weiboData.statuses.forEach(function (detail) {
      if (!detail) return;
      feed.item({
        title: detail.status_title || (detail.text ? detail.text.replace(/<[^>]+>/g, '').replace(/[\n]/g, '').substr(0, 25) : null),
        description: utils.formatStatus(detail, options.largePic, options.emoji),
        url: 'https://weibo.com/' + uid + '/' + detail.bid,
        guid: 'https://weibo.com/' + uid + '/' + detail.bid,
        date: new Date(detail.created_at)
      });
    });
    resultXML = feed.xml();
  }

  return {
    userNotExist,
    requestSuccess,
    resultXML
  };
};

// 通过用户的个性域名获取UID
exports.getUIDByDomain = function (domain) {
  // 利用手机版的跳转获取
  return axiosInstance.get(`https://m.weibo.cn/${domain}?&jumpfrom=weibocom`, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A356 Safari/604.1'
    },
    maxRedirects: 0,
  }).then(res => {
    // fake 404
    return Promise.reject(new Error('User Not Exist'));
  }).catch(err => {
    if (err.response) {
      if (err.response.status === 302) {
        const uid = err.response.headers.location.split("/u/")[1];
        return {
          uid
        };
      } else if (err.response.status === 404) {
        return {
          notFound: true,
        };
      }
    }
    return Promise.reject(err);
  });
};

/**
 * 获取最近发的微博
 */
async function getWeiboData(uid) {
  let resultList = [];
  let tempResultObject = await getByIndexAPI(uid);
  resultList.push(tempResultObject);
  if (!tempResultObject.requestSuccess || tempResultObject.userNotExist) {
    tempResultObject = await getByProfileInfo(uid);
    resultList.push(tempResultObject);
  }

  // result
  let weiboData;
  let userNotExist = resultList.every(item => item.userNotExist);
  let requestSuccess = resultList.some(item => item.requestSuccess);

  // backup cache
  let cacheWeiboData = await cache.get(`wbdata-${uid}`);
  if (!requestSuccess && !tempResultObject.weiboData && cacheWeiboData) {
    logger.info(`bkcache-${uid}`);
    tempResultObject.weiboData = cacheWeiboData;
  } else if (!tempResultObject.userNotExist) {
    cache.set(`wbdata-${uid}`, tempResultObject.weiboData, 86400000);
  }

  if (!tempResultObject.userNotExist) {
    weiboData = await handleLongText(tempResultObject.weiboData);
  }

  return {
    userNotExist,
    requestSuccess,
    weiboData
  };
};

/**
 * 出错时自动暂停
 */
const avaliableMethod = {
  methods: {
    indexAPI: {
      enable: true,
      lastUpdatedTime: Date.now(),
      retryDelay: 600000,
    },
    profileInfo: {
      enable: true,
      lastUpdatedTime: Date.now(),
{'fraction_non_alphanumeric': 0.15786708694510113, 'fraction_numerical': 0.016023115313895456, 'mean_word_length': 2.8911599386816556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4459760', 'n_tokens_mistral': 2735, 'n_tokens_neox': 2623, 'n_words': 517}