<commit_before>/*
 * Copyright 2017 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../third_party/externs/ts/node/node.d.ts" />
/// <reference path="common.d.ts" />

import fs = require('fs');
import http = require('http');
import logging = require('./logging');
import url = require('url');

interface Creds {
  account: string;
  scopes: string;
  access_token:  string;
  expires_in: number;
  token_type: string;
}

interface FakeMetadata {
  project: string;
  project_number: string;
  creds: Creds;
}

const metadata: FakeMetadata = {
  project: process.env.PROJECT_ID,
  project_number: process.env.PROJECT_NUMBER,
  creds: {
    account: process.env.DATALAB_GIT_AUTHOR,
    scopes: "",
    access_token: "",
    expires_in: 0,
    token_type: "Bearer",
  },
};

function launchFakeServer(metadata: FakeMetadata): void {
  const port = 80;
  const metadata_host = '169.254.169.254';
  logging.getLogger().info('Starting fake metadata server at http://%s:%d with %s',
                           metadata_host, port, JSON.stringify(metadata));

  const server = http.createServer((req, res) => {
    const parsed_url = url.parse(req.url, true);
    const urlpath = parsed_url.pathname;
    logging.getLogger().info('Service a fake metadata request at %s', urlpath);

    if (urlpath == '/computeMetadata/v1/project/numeric-project-id') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.project_number);
    } else if (urlpath == '/computeMetadata/v1/project/project-id') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.project);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write('default/\n');
      res.write(metadata.creds.account + '/\n');
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/' &&
               parsed_url.query['recursive'] == "True") {
      const accountJSON: any = {
        aliases: ["default"],
        email: metadata.creds.account,
        scopes: [metadata.creds.scopes],
      };
      res.writeHead(200, { 'Content-Type': 'application/json' });
      res.write(JSON.stringify(accountJSON));
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/email' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/email') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.creds.account);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/scopes' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/scopes') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.creds.scopes);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/token' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/token') {
      const token: any = {
        access_token: metadata.creds.access_token,
        expires_in: metadata.creds.expires_in,
        token_type: metadata.creds.token_type,
      };
      res.writeHead(200, { 'Content-Type': 'application/json' });
      res.write(JSON.stringify(token));
    } else {
      res.writeHead(404);
    }
    res.end();
  });
  server.listen(port, metadata_host, 511, () => {
    // The `gcloud` tool uses a file in its config directory named 'gce' to
    // cache whether or not it should read credentials from the metadata server.
    //
    // Since our fake metadata server just connected, it's possible that an
    // earlier invocation of `gcloud` could have written that file. To account
    // for this, we overwrite the file with the value that indicates the tool
    // should read from the metadata server.
    const gceFile = '/content/datalab/.config/gce'
    fs.writeFileSync(gceFile, "True");
  });
}

/**
 * Initializes the Jupyter server manager.
 */
export function init(settings: common.AppSettings): void {
  if (process.env.DATALAB_FAKE_METADATA_SERVER != 'true') {
    return;
  }
  launchFakeServer(metadata);
}

function parseCreds(request: http.ServerRequest, callback: Function): void {
  let body : string = "";
  request.on('data', function(chunk: string) { body += chunk; });<commit_msg>Make the fake metadata server more general. (#1619)<commit_after>/*
 * Copyright 2017 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/// <reference path="../../../third_party/externs/ts/node/node.d.ts" />
/// <reference path="common.d.ts" />

import fs = require('fs');
import http = require('http');
import logging = require('./logging');
import url = require('url');

interface Creds {
  account: string;
  scopes: string;
  access_token:  string;
  expires_in: number;
  token_type: string;
}

interface FakeMetadata {
  project: string;
  project_number: string;
  creds: Creds;
}

const metadata: FakeMetadata = {
  project: process.env.PROJECT_ID,
  project_number: process.env.PROJECT_NUMBER,
  creds: {
    account: process.env.DATALAB_GIT_AUTHOR,
    scopes: "",
    access_token: "",
    expires_in: 0,
    token_type: "Bearer",
  },
};

function launchFakeServer(metadata: FakeMetadata): void {
  const port = 80;
  const host = 'metadata.google.internal';
  logging.getLogger().info('Starting fake metadata server at http://%s:%d with %s',
                           host, port, JSON.stringify(metadata));

  const server = http.createServer((req, res) => {
    const parsed_url = url.parse(req.url, true);
    const urlpath = parsed_url.pathname;
    logging.getLogger().info('Service a fake metadata request at %s', urlpath);

    if (urlpath == '/computeMetadata/v1/project/numeric-project-id') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.project_number);
    } else if (urlpath == '/computeMetadata/v1/project/project-id') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.project);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write('default/\n');
      res.write(metadata.creds.account + '/\n');
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/' &&
               parsed_url.query['recursive'] == "True") {
      const accountJSON: any = {
        aliases: ["default"],
        email: metadata.creds.account,
        scopes: [metadata.creds.scopes],
      };
      res.writeHead(200, { 'Content-Type': 'application/json' });
      res.write(JSON.stringify(accountJSON));
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/email' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/email') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.creds.account);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/scopes' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/scopes') {
      res.writeHead(200, { 'Content-Type': 'application/text' });
      res.write(metadata.creds.scopes);
    } else if (urlpath == '/computeMetadata/v1/instance/service-accounts/default/token' ||
               urlpath == '/computeMetadata/v1/instance/service-accounts/' + metadata.creds.account + '/token') {
      const token: any = {
        access_token: metadata.creds.access_token,
        expires_in: metadata.creds.expires_in,
        token_type: metadata.creds.token_type,
      };
      res.writeHead(200, { 'Content-Type': 'application/json' });
      res.write(JSON.stringify(token));
    } else {
      res.writeHead(404);
    }
    res.end();
  });
  server.listen(port, host, 511, () => {
    // The `gcloud` tool uses a file in its config directory named 'gce' to
    // cache whether or not it should read credentials from the metadata server.
    //
    // Since our fake metadata server just connected, it's possible that an
    // earlier invocation of `gcloud` could have written that file. To account
    // for this, we overwrite the file with the value that indicates the tool
    // should read from the metadata server.
    const gceFile = '/content/datalab/.config/gce'
    fs.writeFileSync(gceFile, "True");
  });
}

/**
 * Initializes the Jupyter server manager.
 */
export function init(settings: common.AppSettings): void {
  if (process.env.DATALAB_FAKE_METADATA_SERVER != 'true') {
    return;
  }
  launchFakeServer(metadata);
}

function parseCreds(request: http.ServerRequest, callback: Function): void {
  let body : string = "";
  request.on('data', function(chunk: string) { body += chunk; });
{'fraction_non_alphanumeric': 0.14218625099920065, 'fraction_numerical': 0.011191047162270184, 'mean_word_length': 3.2957081545064377, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4716082', 'n_tokens_mistral': 3071, 'n_tokens_neox': 2980, 'n_words': 753}