<commit_before>"use strict";
import moment from "moment";
import axios from "axios";
import Feedparser from "feedparser";
import stringToStream from "string-to-stream";

export function getRssList() {
  return localStorage.getItem("rssList") == null
    ? undefined
    : JSON.parse(localStorage.getItem("rssList"));
}

export function transformDate(timeInMS) {
  var momentDate = moment(timeInMS);
  var momentNow = moment(new Date());
  if (momentNow.diff(momentDate, "days") > 0) {
    return momentDate.format("LL");
  }
  return momentDate.startOf("minute").fromNow();
}

export function feedParse(url) {
  var feedparser = new Feedparser();
  return axios({ method: "get", url: url })
    .then(res => {
      // res.data.pipe(feedparser);
      stringToStream(res.data).pipe(feedparser);
    })
    .then(() => {
      var promise = new Promise((resolve, reject) => {
        let items = [];
        feedparser.on("readable", function() {
          var stream = this;
          var item;
          while ((item = stream.read())) {
            items.push(item);
          }
        });

        feedparser.on("end", () => {<commit_msg>add timeout setting<commit_after>"use strict";
import moment from "moment";
import axios from "axios";
import Feedparser from "feedparser";
import stringToStream from "string-to-stream";

export function getRssList() {
  return localStorage.getItem("rssList") == null
    ? undefined
    : JSON.parse(localStorage.getItem("rssList"));
}

export function transformDate(timeInMS) {
  var momentDate = moment(timeInMS);
  var momentNow = moment(new Date());
  if (momentNow.diff(momentDate, "days") > 0) {
    return momentDate.format("LL");
  }
  return momentDate.startOf("minute").fromNow();
}

export function feedParse(url) {
  var feedparser = new Feedparser();
  return axios({ method: "get", url: url, timeout:3000 })
    .then(res => {
      // res.data.pipe(feedparser);
      stringToStream(res.data).pipe(feedparser);
    })
    .then(() => {
      var promise = new Promise((resolve, reject) => {
        let items = [];
        feedparser.on("readable", function() {
          var stream = this;
          var item;
          while ((item = stream.read())) {
            items.push(item);
          }
        });

        feedparser.on("end", () => {
{'fraction_non_alphanumeric': 0.15377932232841007, 'fraction_numerical': 0.0026064291920069507, 'mean_word_length': 3.068904593639576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1525559', 'n_tokens_mistral': 694, 'n_tokens_neox': 684, 'n_words': 163}