<commit_before>var moment = Moment.load();

function resetScriptState() {
  var userProperties = PropertiesService.getUserProperties();
  userProperties.deleteAllProperties();
  var projectTriggers = ScriptApp.getProjectTriggers();
  for (var i = 0; i < projectTriggers.length; i++) {
    ScriptApp.deleteTrigger(projectTriggers[i]);
  }
  var files = DriveApp.searchFiles("title contains 'Email3 Data'");
  while (files.hasNext()) {
    var file = files.next();
    Logger.log("Trashing: " + file.getName());
    file.setTrashed(true);
  }
}

function emailReport(scriptState, msgConsumers) {
  Logger.log('Sending last months report via Email');
  for(i = 0; i < msgConsumers.length; i += 1) {
    msgConsumers[i].graphFunction();
  }
  var htmlBody = "";
  var inlineImages = {};
  var blob;

  for(i = 0; i < msgConsumers.length; i += 1) {
    htmlBody += msgConsumers[i].name + " <img src='cid:" + i + "'> <br>\n";
    blob = msgConsumers[i].chart.getAs('image/png').setName(msgConsumers[i].name + ".png");
    inlineImages[i] = blob;
  }

  MailApp.sendEmail({
    to: Session.getActiveUser().getEmail(),
    subject: "Email3 - " + scriptState.monthToProcess.format('MMM YYYY'),
    htmlBody: htmlBody,
    inlineImages: inlineImages
  });
  scriptState.reportEmailed = true;
}

function createSpreadSheet(monthToProcess) {
  var ssNew = SpreadsheetApp.create("Email3 Data - " + monthToProcess.format('MMM YYYY'));
  var ssID = ssNew.getUrl().split('/')[7];
  // This is needed so we can use the Google Visualization API Query Language
  // to set the data source URL for graphs
  allowPublicViewing(ssID);
  return ssID;
}

function initializeSpreadSheet(scriptState) {
  var msgConsumers = getMsgConsumers(scriptState);
  for(i = 0; i < msgConsumers.length; i += 1) {
    msgConsumers[i].initSpreadSheet();
  }
}

function populateData(scriptState, msgConsumers) {
  Logger.log('Populating Data');
  // Process up to 100 gmail threads at once
  var batchSize = 100;
  var after = scriptState.monthToProcess.clone().startOf('month').format('YYYY/MM/DD');
  var before = scriptState.monthToProcess.clone().endOf('month').add(1, 'days').format('YYYY/MM/DD');
  var query = "in:anywhere -label:sms -label:call-log -label:chats -label:spam -filename:ics";
  query += " -from:maestro.bounces.google.com -from:unified-notifications.bounces.google.com -from:docs.google.com";
  query += " -from:group.calendar.google.com -from:<EMAIL>";
  query += " -from:sites.bounces.google.com";
  query += " after:" + after + " before:" + before;

  Logger.log('Searching: ' + query);
  var threads = GmailApp.search(query, scriptState.threadIndex, batchSize);
  if (threads.length === 0) {
    scriptState.dataPopulated = true;
  } else {
    for (thread = 0; thread < threads.length; thread += 1) {
      var msg = threads[thread].getMessages()[0];
      for(var i = 0; i < msgConsumers.length; i += 1) {
        msgConsumers[i].msgFunction(msg);
      }
    }
  }
  scriptState.threadIndex += batchSize;
  Logger.log('PopulateData run complete: threads: ' + threads.length + ', threadIndex: ' + scriptState.threadIndex);
}

function setScriptState(scriptState) {
  var scriptStateString = JSON.stringify(scriptState);
  PropertiesService.getUserProperties().setProperty("scriptState", scriptStateString);
  Logger.log("New scriptState set: " + scriptStateString);
}

function getScriptState() {
  var scriptState = null;
  var scriptStateString = PropertiesService.getUserProperties().getProperty("scriptState");
  Logger.log("Retrieved scriptState: " + scriptStateString);
  if (scriptStateString === null) {
    Logger.log('No stored scriptState, creating an empty one');
    scriptState = {};
    var currentMonth = moment().startOf('month');
    var nextMonthToProcess = currentMonth.clone().subtract(1, 'days').startOf('month');
    scriptState.ssID = createSpreadSheet(nextMonthToProcess);
    initializeSpreadSheet(scriptState);
    scriptState.monthToProcess = nextMonthToProcess;
    scriptState.dataPopulated = false;
    scriptState.reportEmailed = false;
    scriptState.threadIndex = 0;
    scriptState.triggers = [];
    scriptState.dailyRunningTimeTotalSeconds = 0;
    scriptState.currentDay = moment().startOf('day');
  } else {
    scriptState = JSON.parse(scriptStateString);
    scriptState.monthToProcess = moment(scriptState.monthToProcess);
    scriptState.currentDay = moment(scriptState.currentDay);
  }
  
  // Reset the running total, if it is a new day
  if (! moment().startOf('day').isSame(scriptState.currentDay)) {
    scriptState.dailyRunningTimeTotalSeconds = 0;
    scriptState.currentDay = moment().startOf('day');
  }

  // Create a daily trigger if it does not exist
  dailyTrigger = underscoreGS._find(scriptState.triggers, function (trigger) {
    return trigger.type === "atHour";
  });
  if (dailyTrigger === undefined) {
    Logger.log("No daily trigger, adding one");
    trigger = ScriptApp.newTrigger("main")
    .timeBased()
    .atHour(1)
    .everyDays(1)
    .create();
    scriptState.triggers.push({
      type: "atHour",
      uniqueId: trigger.getUniqueId(),
    });
  }

  return scriptState;
}

function deleteSheet1(scriptState) {<commit_msg>Add ability to enable debug mode<commit_after>var moment = Moment.load();

function enableDebugMode() {
  PropertiesService.getUserProperties().setProperty("debugEnabled", "true");
  Logger.log("Debugging enabled");
}

function disableDebugMode() {
  PropertiesService.getUserProperties().setProperty("debugEnabled", "false");
  Logger.log("Debugging disabled");
}

function resetScriptState() {
  var userProperties = PropertiesService.getUserProperties();
  userProperties.deleteAllProperties();
  var projectTriggers = ScriptApp.getProjectTriggers();
  for (var i = 0; i < projectTriggers.length; i++) {
    ScriptApp.deleteTrigger(projectTriggers[i]);
  }
  var files = DriveApp.searchFiles("title contains 'Email3 Data'");
  while (files.hasNext()) {
    var file = files.next();
    Logger.log("Trashing: " + file.getName());
    file.setTrashed(true);
  }
}

function emailReport(scriptState, msgConsumers) {
  Logger.log('Sending last months report via Email');
  for(i = 0; i < msgConsumers.length; i += 1) {
    msgConsumers[i].graphFunction();
  }
  var htmlBody = "";
  var inlineImages = {};
  var blob;

  for(i = 0; i < msgConsumers.length; i += 1) {
    htmlBody += msgConsumers[i].name + " <img src='cid:" + i + "'> <br>\n";
    blob = msgConsumers[i].chart.getAs('image/png').setName(msgConsumers[i].name + ".png");
    inlineImages[i] = blob;
  }

  MailApp.sendEmail({
    to: Session.getActiveUser().getEmail(),
    subject: "Email3 - " + scriptState.monthToProcess.format('MMM YYYY'),
    htmlBody: htmlBody,
    inlineImages: inlineImages
  });
  scriptState.reportEmailed = true;
}

function createSpreadSheet(monthToProcess) {
  var ssNew = SpreadsheetApp.create("Email3 Data - " + monthToProcess.format('MMM YYYY'));
  var ssID = ssNew.getUrl().split('/')[7];
  // This is needed so we can use the Google Visualization API Query Language
  // to set the data source URL for graphs
  allowPublicViewing(ssID);
  return ssID;
}

function initializeSpreadSheet(scriptState) {
  var msgConsumers = getMsgConsumers(scriptState);
  for(i = 0; i < msgConsumers.length; i += 1) {
    msgConsumers[i].initSpreadSheet();
  }
}

function populateData(scriptState, msgConsumers) {
  Logger.log('Populating Data');
  // Process up to 100 gmail threads at once
  var batchSize = 100;
  var after = scriptState.monthToProcess.clone().startOf('month').format('YYYY/MM/DD');
  var before = scriptState.monthToProcess.clone().endOf('month').add(1, 'days').format('YYYY/MM/DD');
  var query = "in:anywhere -label:sms -label:call-log -label:chats -label:spam -filename:ics";
  query += " -from:maestro.bounces.google.com -from:unified-notifications.bounces.google.com -from:docs.google.com";
  query += " -from:group.calendar.google.com -from:<EMAIL>";
  query += " -from:sites.bounces.google.com";
  query += " after:" + after + " before:" + before;

  Logger.log('Searching: ' + query);
  var threads = GmailApp.search(query, scriptState.threadIndex, batchSize);
  if (scriptState.debugEnabled && scriptState.threadIndex > 200) {
    scriptState.dataPopulated = true;
  } else if (threads.length === 0) {
    scriptState.dataPopulated = true;
  } else {
    for (thread = 0; thread < threads.length; thread += 1) {
      var msg = threads[thread].getMessages()[0];
      for(var i = 0; i < msgConsumers.length; i += 1) {
        if (scriptState.debugEnabled) {
          Logger.log('Calling Message function for, ' + msgConsumers[i].name);
        }
        msgConsumers[i].msgFunction(msg);
      }
    }
  }
  scriptState.threadIndex += batchSize;
  Logger.log('PopulateData run complete: threads: ' + threads.length + ', threadIndex: ' + scriptState.threadIndex);
}

function setScriptState(scriptState) {
  var scriptStateString = JSON.stringify(scriptState);
  PropertiesService.getUserProperties().setProperty("scriptState", scriptStateString);
  Logger.log("New scriptState set: " + scriptStateString);
}

function getScriptState() {
  var scriptState = null;
  var nextMonthToProcess;
  var scriptStateString = PropertiesService.getUserProperties().getProperty("scriptState");
  var debugEnabledUserProperty = PropertiesService.getUserProperties().getProperty("debugEnabled");
  if (debugEnabledUserProperty === "true") {
    debugEnabled = true;
    Logger.log("Debugging enabled");
  } else {
    debugEnabled = false;
    Logger.log("Debugging disabled");
  }
  Logger.log("Retrieved scriptState: " + scriptStateString);
  if (scriptStateString === null) {
    Logger.log('No stored scriptState, creating an empty one');
    scriptState = {};
    scriptState.debugEnabled = debugEnabled;
    var currentMonth = moment().startOf('month');
    if (scriptState.debugEnabled) {
      nextMonthToProcess = currentMonth.clone().startOf('year');
    } else {
      nextMonthToProcess = currentMonth.clone().subtract(1, 'days').startOf('month');
    }
    scriptState.ssID = createSpreadSheet(nextMonthToProcess);
    initializeSpreadSheet(scriptState);
    scriptState.monthToProcess = nextMonthToProcess;
    scriptState.dataPopulated = false;
    scriptState.reportEmailed = false;
    scriptState.threadIndex = 0;
    scriptState.triggers = [];
    scriptState.dailyRunningTimeTotalSeconds = 0;
    scriptState.currentDay = moment().startOf('day');
  } else {
    scriptState = JSON.parse(scriptStateString);
    scriptState.debugEnabled = debugEnabled;
    scriptState.monthToProcess = moment(scriptState.monthToProcess);
    scriptState.currentDay = moment(scriptState.currentDay);
  }
  
  // Reset the running total, if it is a new day
  if (! moment().startOf('day').isSame(scriptState.currentDay)) {
    scriptState.dailyRunningTimeTotalSeconds = 0;
    scriptState.currentDay = moment().startOf('day');
  }

  // Create a daily trigger if it does not exist
  dailyTrigger = underscoreGS._find(scriptState.triggers, function (trigger) {
    return trigger.type === "atHour";
  });
  if (dailyTrigger === undefined) {
    Logger.log("No daily trigger, adding one");
    trigger = ScriptApp.newTrigger("main")
    .timeBased()
    .atHour(1)
    .everyDays(1)
    .create();
    scriptState.triggers.push({
      type: "atHour",
      uniqueId: trigger.getUniqueId(),
    });
  }

  return scriptState;
}

function deleteSheet1(scriptState) {
{'fraction_non_alphanumeric': 0.14121037463976946, 'fraction_numerical': 0.0056763601432189325, 'mean_word_length': 4.243589743589744, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1096841', 'n_tokens_mistral': 3579, 'n_tokens_neox': 3509, 'n_words': 728}