<commit_before>/*
Script to fetch movie showtimes from SF Cinemacity,
as well as the meta data for each movie, and store that data to Firebase.
*/

// npm libraries
const admin = require('firebase-admin');
const sfcinemacity = require('sfcinemacity');

// modules
const movieDatabases = require('./modules/movie-databases');

if (!process.env.MOVIES_DATABASE_URL) {
  console.log(`The MOVIES_DATABASE_URL environment variable cannot be found.
Aborting...`);
  return;
}

// setup Firebase
const firebaseApp = admin.initializeApp({
  credential: admin.credential.cert('./private-key.json'),
  databaseURL: process.env.MOVIES_DATABASE_URL
});

const db = admin.database();
const ref = db.ref('/');

const updateMovieDB = async () => {
  try {
    console.log(
      `Updating Maya Mall movie database information at ${new Date().toString()}`
    );

    const mayaMallId = 9936;

    // first wipe out all the data to minimise the data downloaded on the client
    await ref.remove();

    const totalDaysToFetch = 3;

    for (let dayOffset = 0; dayOffset < totalDaysToFetch; dayOffset += 1) {
      // get the showtimes for a specific day
      const showtimes = await sfcinemacity.getShowtimes(mayaMallId, dayOffset);

      console.log(
        `Getting showtimes for ${showtimes.movieTheatreName} for ${
          showtimes.date
        }`
      );

      // check if showtimes were returned
      if (showtimes.movies.length !== 0) {
        // save that showtime info to Firebase
        // await ref
        //   .child(`movie-theatres/chiangmai/${mayaMallId}/${showtimes.date}`)
        //   .set(showtimes.movies);
        console.log(`Saved showtime data.`);

        // go through each movie and save the meta data for it to Firebase
        console.log('Updating movie metadata for this day...');
        for (let movie of showtimes.movies) {
          // replace illegal Firebase keys with a -
          const movieTitle = movie.movieTitle.replace(/\.|\$|\[|\]|#|\//g, '-');

          // save the showtime info to Firebase
          await ref
            .child(
              `movie-theatres/chiangmai/${mayaMallId}/${
                showtimes.date
              }/${movieTitle}`
            )
            .set(movie.cinemas);

          let movieData = {
            rating: movie.rating
          };

          const movieDbData = await movieDatabases.theMovieDB(movieTitle);
          movieData = Object.assign(movieData, movieDbData);
          const rottenTomatoesData = await movieDatabases.rottenTomatoes(
            movieTitle
          );
          movieData = Object.assign(movieData, rottenTomatoesData);

          await ref.child(`movie-details/${movieTitle}`).set(movieData);
          console.log(`Updated movie data for ${movieTitle}.`);
        }
      } else {
        console.log('No showtimes found.');
      }
    }
    console.log('Data scraping complete.');<commit_msg>differentiating firebase key and movie title<commit_after>/*
Script to fetch movie showtimes from SF Cinemacity,
as well as the meta data for each movie, and store that data to Firebase.
*/

// npm libraries
const admin = require('firebase-admin');
const sfcinemacity = require('sfcinemacity');

// modules
const movieDatabases = require('./modules/movie-databases');

if (!process.env.MOVIES_DATABASE_URL) {
  console.log(`The MOVIES_DATABASE_URL environment variable cannot be found.
Aborting...`);
  return;
}

// setup Firebase
const firebaseApp = admin.initializeApp({
  credential: admin.credential.cert('./private-key.json'),
  databaseURL: process.env.MOVIES_DATABASE_URL
});

const db = admin.database();
const ref = db.ref('/');

const updateMovieDB = async () => {
  try {
    console.log(
      `Updating Maya Mall movie database information at ${new Date().toString()}`
    );

    const mayaMallId = 9936;

    // first wipe out all the data to minimise the data downloaded on the client
    await ref.remove();

    const totalDaysToFetch = 3;

    for (let dayOffset = 0; dayOffset < totalDaysToFetch; dayOffset += 1) {
      // get the showtimes for a specific day
      const showtimes = await sfcinemacity.getShowtimes(mayaMallId, dayOffset);

      console.log(
        `Getting showtimes for ${showtimes.movieTheatreName} for ${
          showtimes.date
        }`
      );

      // check if showtimes were returned
      if (showtimes.movies.length !== 0) {
        // save that showtime info to Firebase
        // await ref
        //   .child(`movie-theatres/chiangmai/${mayaMallId}/${showtimes.date}`)
        //   .set(showtimes.movies);
        console.log(`Saved showtime data.`);

        // go through each movie and save the meta data for it to Firebase
        console.log('Updating movie metadata for this day...');
        for (let movie of showtimes.movies) {
          // replace illegal Firebase keys with a -
          const movieKey = movie.movieTitle.replace(/\.|\$|\[|\]|#|\//g, '-');

          // save the showtime info to Firebase
          await ref
            .child(
              `movie-theatres/chiangmai/${mayaMallId}/${
                showtimes.date
              }/${movieKey}`
            )
            .set(movie.cinemas);

          let movieData = {
            rating: movie.rating
          };

          const movieDbData = await movieDatabases.theMovieDB(movie.movieTitle);
          movieData = Object.assign(movieData, movieDbData);
          const rottenTomatoesData = await movieDatabases.rottenTomatoes(
            movie.movieTitle
          );
          movieData = Object.assign(movieData, rottenTomatoesData);

          await ref.child(`movie-details/${movieKey}`).set(movieData);
          console.log(`Updated movie data for ${movie.movieTitle}.`);
        }
      } else {
        console.log('No showtimes found.');
      }
    }
    console.log('Data scraping complete.');
{'fraction_non_alphanumeric': 0.11793821471240826, 'fraction_numerical': 0.00273084144051886, 'mean_word_length': 2.3971014492753624, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '952741', 'n_tokens_mistral': 1751, 'n_tokens_neox': 1671, 'n_words': 456}