<commit_before>const Webtorrent = require('webtorrent');
const pump = require('pump');
const mime = require('mime');
const rangeParser = require('range-parser');
const admin = require("firebase-admin");
const firestore = admin.firestore();
const _ = require('lodash');

const webtorrent = new Webtorrent();
const listTorrent = {}

// From https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
function encodeRFC5987 (str) {
    return encodeURIComponent(str)
      // Note that although RFC3986 reserves "!", RFC5987 does not,
      // so we do not need to escape it
      .replace(/['()]/g, escape) // i.e., %27 %28 %29
      .replace(/\*/g, '%2A')
      // The following are not required for percent-encoding per RFC5987,
      // so we can allow for a little better readability over the wire: |`^
      .replace(/%(?:7C|60|5E)/g, unescape)
  }

class Torrent {
  
    constructor() {
        firestore.collection('session')
            .onSnapshot(snapshot => {
                snapshot.docChanges.forEach(change => {
                  const torrentUrl = change.doc.data().torrentUrl;
                  const limitDate = new Date();
                  limitDate.setHours(limitDate.getHours() - 3);
                  console.log(`added on ${change.doc.data().date} with limit ${limitDate.getTime()}`)
                  if (change.type === 'added') {
                    if (change.doc.data().date > limitDate.getTime()) {
                      this.addNewTorrent(change.doc.id, torrentUrl);
                    }
                  }
                });
            });
        
        webtorrent.on('error', err => {
          console.log(err);
        });

        setInterval(this.expiredTorrent, 3600 * 1000);
    }

    expiredTorrent() {
      console.log(`Destroying old torrent`);
      const expiredDate = new Date();
      expiredDate.setHours(expiredDate.getHours()-3);
      _.keys(listTorrent).forEach(url => {
        const torr = listTorrent[url];
        if (torr.date < expiredDate.getTime()) {
          //  torr is not accessed in last 3 hours
          torr.torrent.destroy(() => {
            console.log(`torrent ${url} is destroyed`);
            delete listTorrent[url];
          });
        }
      });
    }

    addNewTorrent(sessionId, torrentFile) {
      console.log(listTorrent[torrentFile]);
      if (listTorrent[torrentFile]) {
        return this.notifySessionReady(sessionId);
      }
      console.log(`adding ${torrentFile}`);
      webtorrent.add(torrentFile, (torrent) => {
            console.log(`Torrent for ${sessionId} is available`);
            const date = new Date();
            listTorrent[torrentFile] = {
              torrent,
              date: date.getTime(),
            };
            this.notifySessionReady(sessionId);
        });
    }

    notifySessionReady(sessionId) {
      firestore.doc(`/session/${sessionId}`).update({
        status: 'ready',
      }).then(() => {
        console.log(`URL ${sessionId} is ready`);<commit_msg>remove logs<commit_after>const Webtorrent = require('webtorrent');
const pump = require('pump');
const mime = require('mime');
const rangeParser = require('range-parser');
const admin = require("firebase-admin");
const firestore = admin.firestore();
const _ = require('lodash');

const webtorrent = new Webtorrent();
const listTorrent = {}

// From https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
function encodeRFC5987 (str) {
    return encodeURIComponent(str)
      // Note that although RFC3986 reserves "!", RFC5987 does not,
      // so we do not need to escape it
      .replace(/['()]/g, escape) // i.e., %27 %28 %29
      .replace(/\*/g, '%2A')
      // The following are not required for percent-encoding per RFC5987,
      // so we can allow for a little better readability over the wire: |`^
      .replace(/%(?:7C|60|5E)/g, unescape)
  }

class Torrent {
  
    constructor() {
        firestore.collection('session')
            .onSnapshot(snapshot => {
                snapshot.docChanges.forEach(change => {
                  const torrentUrl = change.doc.data().torrentUrl;
                  const limitDate = new Date();
                  limitDate.setHours(limitDate.getHours() - 3);
                  console.log(`added on ${change.doc.data().date} with limit ${limitDate.getTime()}`)
                  if (change.type === 'added') {
                    if (change.doc.data().date > limitDate.getTime()) {
                      this.addNewTorrent(change.doc.id, torrentUrl);
                    }
                  }
                });
            });
        
        webtorrent.on('error', err => {
          console.log(err);
        });

        setInterval(this.expiredTorrent, 3600 * 1000);
    }

    expiredTorrent() {
      console.log(`Destroying old torrent`);
      const expiredDate = new Date();
      expiredDate.setHours(expiredDate.getHours()-3);
      _.keys(listTorrent).forEach(url => {
        const torr = listTorrent[url];
        if (torr.date < expiredDate.getTime()) {
          //  torr is not accessed in last 3 hours
          torr.torrent.destroy(() => {
            console.log(`torrent ${url} is destroyed`);
            delete listTorrent[url];
          });
        }
      });
    }

    addNewTorrent(sessionId, torrentFile) {
      if (listTorrent[torrentFile]) {
        return this.notifySessionReady(sessionId);
      }
      console.log(`adding ${torrentFile}`);
      webtorrent.add(torrentFile, (torrent) => {
            console.log(`Torrent for ${sessionId} is available`);
            const date = new Date();
            listTorrent[torrentFile] = {
              torrent,
              date: date.getTime(),
            };
            this.notifySessionReady(sessionId);
        });
    }

    notifySessionReady(sessionId) {
      firestore.doc(`/session/${sessionId}`).update({
        status: 'ready',
      }).then(() => {
        console.log(`URL ${sessionId} is ready`);
{'fraction_non_alphanumeric': 0.14146099055822428, 'fraction_numerical': 0.012589034288553918, 'mean_word_length': 2.1059670781893005, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1732958', 'n_tokens_mistral': 1814, 'n_tokens_neox': 1739, 'n_words': 359}