<commit_before>/*
 * Copyright 2015 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../third_party/externs/ts/node/node.d.ts" />
/// <reference path="../../../third_party/externs/ts/node/node-uuid.d.ts" />
/// <reference path="common.d.ts" />

import childProcess = require('child_process');
import fs = require('fs');
import http = require('http');
import uuid = require('node-uuid');
import path = require('path');
import querystring = require('querystring');
import url = require('url');
import util = require('util');
import idleTimeout = require('./idleTimeout');
import userManager = require('./userManager');

var SETTINGS_FILE = 'settings.json';
var DEFAULT_USER_SETTINGS_FILE = 'userSettings.json';
var METADATA_FILE = 'metadata.json';
const IDLE_TIMEOUT_KEY = 'idleTimeoutInterval';

interface Metadata {
  instanceId: string;
}

/**
 * Loads the configuration settings for the application to use.
 * On first run, this generates any dynamic settings and merges them into the settings result.
 * @returns the settings object for the application to use.
 */
export function loadAppSettings(): common.AppSettings {
  var settingsPath = path.join(__dirname, 'config', SETTINGS_FILE);
  var metadataPath = path.join(__dirname, 'config', METADATA_FILE);

  if (!fs.existsSync(settingsPath)) {
    console.log('App settings file %s not found.', settingsPath);
    return null;
  }

  try {
    var metadata: Metadata = null;
    if (!fs.existsSync(metadataPath)) {
      // Create an write out metadata on the first run if it doesn't exist.
      metadata = { instanceId: uuid.v4() };
      fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2), { encoding: 'utf8' });
    }
    else {
      // Load metadata from the file system. This is written out on the first run.
      metadata = <Metadata>JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
    }

    var settings = <common.AppSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
    settings.versionId = process.env['DATALAB_VERSION'] || '';
    if (process.env['DATALAB_CONFIG_URL']) {
      settings.configUrl = process.env['DATALAB_CONFIG_URL'];
    }
    const settingsOverrides = process.env['DATALAB_SETTINGS_OVERRIDES'];
    if (settingsOverrides) {
      // Allow overriding individual settings via JSON provided as an environment variable.
      const overrides = JSON.parse(settingsOverrides);
      for (const key of Object.keys(overrides)) {
        (<any>settings)[key] = overrides[key];
      }
    }

    // Normalize the base path to include "/" characters.
    if (settings.datalabBasePath.indexOf("/") != 0) {
      settings.datalabBasePath = "/" + settings.datalabBasePath;
    }
    if (settings.datalabBasePath.lastIndexOf("/") != settings.datalabBasePath.length - 1) {
      settings.datalabBasePath = settings.datalabBasePath + "/";
    }
    return settings;
  }
  catch (e) {
    console.log(e);
    return null;
  }
}

/**
 * Loads the path of the configuration directory for the user.
 *
 * @returns the path of the user's config directory.
 */
export function getUserConfigDir(userId: string): string {
  var userDir = userManager.getUserDir(userId);
  var configPath = path.join(userDir, 'datalab', '.config');
  return configPath;
}

/**
 * Copies the default user settings into the user's directory.
 */
function copyDefaultUserSettings(userId: string) {
  var userSettingsPath = path.join(getUserConfigDir(userId), SETTINGS_FILE);
  const defaultUserSettingsPath = path.join(__dirname, 'config', DEFAULT_USER_SETTINGS_FILE);
  console.log('Copying default settings: ' + defaultUserSettingsPath + ' to: ' + userSettingsPath);
  // Copy the default user settings file into user's directory.
  const data = fs.readFileSync(defaultUserSettingsPath);
  fs.writeFileSync(userSettingsPath, data);
}

/**
 * Loads the configuration settings for the user.
 *
 * @returns the key:value mapping of settings for the user.
 */
export function loadUserSettings(userId: string): common.UserSettings {
  var settingsPath = path.join(getUserConfigDir(userId), SETTINGS_FILE);
  if (!fs.existsSync(settingsPath)) {
    console.log('User settings file %s not found, copying default settings.', settingsPath);
    copyDefaultUserSettings(userId);
  }

  try {
    var settings = <common.UserSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
    return settings;
  }
  catch (e) {
    console.log(e);
    return null;
  }
}

function ensureDirExists(fullPath: string): boolean {
  if (path.dirname(fullPath) == fullPath) {
    // This should only happen once we hit the root directory
    return true;
  }
  if (fs.existsSync(fullPath)) {
    if (!fs.lstatSync(fullPath).isDirectory()) {
      console.log('Path ' + fullPath + ' is not a directory');
      return false;
    }
    return true;
  }
  if (!ensureDirExists(path.dirname(fullPath))) {
    return false;
  }
  fs.mkdirSync(fullPath);
  return true;
}

/**
 * Updates a single configuration setting for the user.
 *
 * @param key the name of the setting to update.
 * @param value the updated value of the setting.
 * @returns true iff the update was applied.
 */
export function updateUserSetting(userId: string, key: string, value: string,
                                  asynchronous: boolean = false): boolean {
  var userDir = userManager.getUserDir(userId);
  var settingsDir =  path.join(userDir, 'datalab', '.config');
  var settingsPath = path.join(settingsDir, SETTINGS_FILE);

  if (!fs.existsSync(settingsPath)) {
    console.log('User settings file %s not found, copying default settings.', settingsPath);
    copyDefaultUserSettings(userId);
  }

  let settings: common.UserSettings;
  if (fs.existsSync(settingsPath)) {
    try {
      settings = <common.UserSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
    }
    catch (e) {
      console.log(e);
      return false;
    }
  }
  settings[key] = value;

  try {
    var settingsString = JSON.stringify(settings);<commit_msg>Avoid failing if empty settings file found (#1467)<commit_after>/*
 * Copyright 2015 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../third_party/externs/ts/node/node.d.ts" />
/// <reference path="../../../third_party/externs/ts/node/node-uuid.d.ts" />
/// <reference path="common.d.ts" />

import childProcess = require('child_process');
import fs = require('fs');
import http = require('http');
import uuid = require('node-uuid');
import path = require('path');
import querystring = require('querystring');
import url = require('url');
import util = require('util');
import idleTimeout = require('./idleTimeout');
import userManager = require('./userManager');

var SETTINGS_FILE = 'settings.json';
var DEFAULT_USER_SETTINGS_FILE = 'userSettings.json';
var METADATA_FILE = 'metadata.json';
const IDLE_TIMEOUT_KEY = 'idleTimeoutInterval';

interface Metadata {
  instanceId: string;
}

/**
 * Loads the configuration settings for the application to use.
 * On first run, this generates any dynamic settings and merges them into the settings result.
 * @returns the settings object for the application to use.
 */
export function loadAppSettings(): common.AppSettings {
  var settingsPath = path.join(__dirname, 'config', SETTINGS_FILE);
  var metadataPath = path.join(__dirname, 'config', METADATA_FILE);

  if (!fs.existsSync(settingsPath)) {
    console.log('App settings file %s not found.', settingsPath);
    return null;
  }

  try {
    var metadata: Metadata = null;
    if (!fs.existsSync(metadataPath)) {
      // Create an write out metadata on the first run if it doesn't exist.
      metadata = { instanceId: uuid.v4() };
      fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2), { encoding: 'utf8' });
    }
    else {
      // Load metadata from the file system. This is written out on the first run.
      metadata = <Metadata>JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
    }

    const settings = <common.AppSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8') || '{}');
    settings.versionId = process.env['DATALAB_VERSION'] || '';
    if (process.env['DATALAB_CONFIG_URL']) {
      settings.configUrl = process.env['DATALAB_CONFIG_URL'];
    }
    const settingsOverrides = process.env['DATALAB_SETTINGS_OVERRIDES'];
    if (settingsOverrides) {
      // Allow overriding individual settings via JSON provided as an environment variable.
      const overrides = JSON.parse(settingsOverrides);
      for (const key of Object.keys(overrides)) {
        (<any>settings)[key] = overrides[key];
      }
    }

    // Normalize the base path to include "/" characters.
    if (settings.datalabBasePath.indexOf("/") != 0) {
      settings.datalabBasePath = "/" + settings.datalabBasePath;
    }
    if (settings.datalabBasePath.lastIndexOf("/") != settings.datalabBasePath.length - 1) {
      settings.datalabBasePath = settings.datalabBasePath + "/";
    }
    return settings;
  }
  catch (e) {
    console.log(e);
    return null;
  }
}

/**
 * Loads the path of the configuration directory for the user.
 *
 * @returns the path of the user's config directory.
 */
export function getUserConfigDir(userId: string): string {
  var userDir = userManager.getUserDir(userId);
  var configPath = path.join(userDir, 'datalab', '.config');
  return configPath;
}

/**
 * Copies the default user settings into the user's directory.
 */
function copyDefaultUserSettings(userId: string) {
  var userSettingsPath = path.join(getUserConfigDir(userId), SETTINGS_FILE);
  const defaultUserSettingsPath = path.join(__dirname, 'config', DEFAULT_USER_SETTINGS_FILE);
  console.log('Copying default settings: ' + defaultUserSettingsPath + ' to: ' + userSettingsPath);
  // Copy the default user settings file into user's directory.
  const data = fs.readFileSync(defaultUserSettingsPath);
  fs.writeFileSync(userSettingsPath, data);
}

/**
 * Loads the configuration settings for the user.
 *
 * @returns the key:value mapping of settings for the user.
 */
export function loadUserSettings(userId: string): common.UserSettings {
  var settingsPath = path.join(getUserConfigDir(userId), SETTINGS_FILE);
  if (!fs.existsSync(settingsPath)) {
    console.log('User settings file %s not found, copying default settings.', settingsPath);
    copyDefaultUserSettings(userId);
  }

  try {
    const settings = <common.UserSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8') || '{}');
    return settings;
  }
  catch (e) {
    console.log(e);
    return null;
  }
}

function ensureDirExists(fullPath: string): boolean {
  if (path.dirname(fullPath) == fullPath) {
    // This should only happen once we hit the root directory
    return true;
  }
  if (fs.existsSync(fullPath)) {
    if (!fs.lstatSync(fullPath).isDirectory()) {
      console.log('Path ' + fullPath + ' is not a directory');
      return false;
    }
    return true;
  }
  if (!ensureDirExists(path.dirname(fullPath))) {
    return false;
  }
  fs.mkdirSync(fullPath);
  return true;
}

/**
 * Updates a single configuration setting for the user.
 *
 * @param key the name of the setting to update.
 * @param value the updated value of the setting.
 * @returns true iff the update was applied.
 */
export function updateUserSetting(userId: string, key: string, value: string,
                                  asynchronous: boolean = false): boolean {
  var userDir = userManager.getUserDir(userId);
  var settingsDir =  path.join(userDir, 'datalab', '.config');
  var settingsPath = path.join(settingsDir, SETTINGS_FILE);

  if (!fs.existsSync(settingsPath)) {
    console.log('User settings file %s not found, copying default settings.', settingsPath);
    copyDefaultUserSettings(userId);
  }

  let settings: common.UserSettings;
  if (fs.existsSync(settingsPath)) {
    try {
      settings = <common.UserSettings>JSON.parse(fs.readFileSync(settingsPath, 'utf8') || '{}');
    }
    catch (e) {
      console.log(e);
      return false;
    }
  }
  settings[key] = value;

  try {
    var settingsString = JSON.stringify(settings);
{'fraction_non_alphanumeric': 0.12283908308582743, 'fraction_numerical': 0.002893915162592339, 'mean_word_length': 3.8067349926793557, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6792151', 'n_tokens_mistral': 3841, 'n_tokens_neox': 3806, 'n_words': 1120}