<commit_before>#!/usr/bin/env node

const crypto = require('crypto');
const fs = require('fs');
const http = require('http');
const path = require('path');
const urlParse = require('url').parse;

const recrawler = require('recrawler');
const request = require('request');
const static = require('node-static');

const rootDir = path.join(__dirname, 'public');
const cacheDir = path.join(rootDir, '.cache');
const host = process.env.TUNNELRUN_FLICKR_HOST || process.env.HOST || '0.0.0.0';
const port = process.env.TUNNELRUN_FLICKR_PORT || process.env.PORT || 7000;

try {
  fs.mkdirSync(cacheDir);
} catch (e) {
}

var staticServer = new static.Server(rootDir, {
  headers: {
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Methods': 'GET',
    'Access-Control-Allow-Headers': 'Content-Type'
  }
});

const server = http.createServer((req, res) => {
  const reqPathame = urlParse(req.url).pathname;

  // Example URL formats:
  // - https://www.flickr.com/photos/hamburgerjung/34539117106
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/in/pool-equirectangular
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/in/pool-equirectangular/
  let flickrUrl = req.url.substr(req.url.indexOf('?') + 1);
  let matches = flickrUrl.match(/.*\/([^\/]+)\/(\d+)/i);
  let photoUser;
  let photoId;

  if (matches) {
    photoUser = matches[1];
    photoId = parseInt(matches[2], 10);
  }

  if (!photoUser || !photoId) {
    staticServer.serve(req, res);
    return;
  }

  if (photoUser && photoId) {
    const hash = crypto.createHash('sha1').update(flickrDownloadUrl).digest('hex');
    const hashPath = hash + '.jpg';
    const hashUrl = '/.cache/' + hashPath;
    const fn = path.join(cacheDir, hashPath);

    fs.stat(fn, function (err, stats) {
      if (!err && stats.isFile()) {
        req.url = hashUrl;
        // Read from file.
        staticServer.serve(req, res);
        return;
      }

      // Fetch from network.<commit_msg>fix missing variable<commit_after>#!/usr/bin/env node

const crypto = require('crypto');
const fs = require('fs');
const http = require('http');
const path = require('path');
const urlParse = require('url').parse;

const recrawler = require('recrawler');
const request = require('request');
const static = require('node-static');

const rootDir = path.join(__dirname, 'public');
const cacheDir = path.join(rootDir, '.cache');
const host = process.env.TUNNELRUN_FLICKR_HOST || process.env.HOST || '0.0.0.0';
const port = process.env.TUNNELRUN_FLICKR_PORT || process.env.PORT || 7000;

try {
  fs.mkdirSync(cacheDir);
} catch (e) {
}

var staticServer = new static.Server(rootDir, {
  headers: {
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Methods': 'GET',
    'Access-Control-Allow-Headers': 'Content-Type'
  }
});

const server = http.createServer((req, res) => {
  const reqPathame = urlParse(req.url).pathname;

  // Example URL formats:
  // - https://www.flickr.com/photos/hamburgerjung/34539117106
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/in/pool-equirectangular
  // - https://www.flickr.com/photos/hamburgerjung/34539117106/in/pool-equirectangular/
  let flickrUrl = req.url.substr(req.url.indexOf('?') + 1);
  let matches = flickrUrl.match(/.*\/([^\/]+)\/(\d+)/i);
  let photoUser;
  let photoId;

  if (matches) {
    photoUser = matches[1];
    photoId = parseInt(matches[2], 10);
  }

  if (!photoUser || !photoId) {
    staticServer.serve(req, res);
    return;
  }

  if (photoUser && photoId) {
    const flickrDownloadUrl = 'https://www.flickr.com/photos/' + photoUser + '/' + photoId + '/sizes/o/';
    const hash = crypto.createHash('sha1').update(flickrDownloadUrl).digest('hex');
    const hashPath = hash + '.jpg';
    const hashUrl = '/.cache/' + hashPath;
    const fn = path.join(cacheDir, hashPath);

    fs.stat(fn, function (err, stats) {
      if (!err && stats.isFile()) {
        req.url = hashUrl;
        // Read from file.
        staticServer.serve(req, res);
        return;
      }

      // Fetch from network.
{'fraction_non_alphanumeric': 0.17759237187127533, 'fraction_numerical': 0.02765196662693683, 'mean_word_length': 3.4308342133051744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 9, 'lorem ipsum': 0, 'www.': 9, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3532005', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1444, 'n_words': 271}