<commit_before>#!/usr/bin/env node
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

const camelCase = require('camelcase');
const commander = require('commander');
const fs = require('fs');
const YAML = require('yaml');
const readline = require('readline');
const {google} = require('googleapis');

const DataSource = require('./data_source.js');
const GoogleClient = require('./google_client.js');
const GoogleDrive = require('./google_drive.js');
const TemplateEngine = require('./template_engine.js');

const SCOPES = ['https://www.googleapis.com/auth/spreadsheets',
				'https://www.googleapis.com/auth/spreadsheets.readonly',
				'https://www.googleapis.com/auth/drive',
				'https://www.googleapis.com/auth/drive.file',
				'https://spreadsheets.google.com/feeds'];

var settings = {};
var googleClient;
var googleDrive;
var dataSource;
var templateEngine;

async function processRecords() {
	console.log('iterate over each record...');

	for (let [index, record] of dataSource.record_array.entries()) {
		// see if a file has already been generated
		if (dataSource.lookup.file_index &&
			record[dataSource.file_header]) {
			if (!settings.replace) {
				console.log('skipping record #' + index);
				continue;
			}
		}

		console.log('processing record #' + index);

		// load template from record if available
		var template;
		if (dataSource.lookup.template_index &&
			record[dataSource.template_header]) {
			template = record[dataSource.template_header];
		}

		let instance = templateEngine.createInstance({
			record: record,
			template: template
		});

		// create template instance for this record
		await instance.initialize();
		await instance.render();
		let file = await instance.createFile();

		// upload file
		if (settings.upload)
			file = await googleDrive.uploadFile(file);

		// save reference to file
		if (settings.change)
			dataSource.saveFileReference(index, file);

		await instance.cleanup();
	}
}

async function main() {
	// load settings
	console.log('load settings and command line arguments...');
	settings = configure();
	console.log('settings loaded:');
	console.log(settings);

	// load Google Client
	console.log('initialize OAuth client...');
	googleClient = new GoogleClient({
		credentials_file: settings.credentials_file,
		token_file: settings.token_file,
		scopes: SCOPES
	});
	await googleClient.initialize();

	// load Google Drive
	console.log('initializing drive connection...');
	googleDrive = new GoogleDrive({
		upload: settings.upload,
		replace: settings.replace,
		google_client: googleClient,
		folder_id: settings.google_folder_id
	});
	await googleDrive.initialize();

	// load data source
	console.log('initialize data source...');
	dataSource = new DataSource({
		template_header: settings.template_header,
		file_header: settings.file_header,
		change: settings.change,
		csv_file: settings.csv_file,<commit_msg>only load google client and connect to drive if necessary<commit_after>#!/usr/bin/env node
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

const camelCase = require('camelcase');
const commander = require('commander');
const fs = require('fs');
const YAML = require('yaml');
const readline = require('readline');
const {google} = require('googleapis');

const DataSource = require('./data_source.js');
const GoogleClient = require('./google_client.js');
const GoogleDrive = require('./google_drive.js');
const TemplateEngine = require('./template_engine.js');

const SCOPES = ['https://www.googleapis.com/auth/spreadsheets',
				'https://www.googleapis.com/auth/spreadsheets.readonly',
				'https://www.googleapis.com/auth/drive',
				'https://www.googleapis.com/auth/drive.file',
				'https://spreadsheets.google.com/feeds'];

var settings = {};
var googleClient;
var googleDrive;
var dataSource;
var templateEngine;

async function processRecords() {
	console.log('iterate over each record...');

	for (let [index, record] of dataSource.record_array.entries()) {
		// see if a file has already been generated
		if (dataSource.lookup.file_index &&
			record[dataSource.file_header]) {
			if (!settings.replace) {
				console.log('skipping record #' + index);
				continue;
			}
		}

		console.log('processing record #' + index);

		// load template from record if available
		var template;
		if (dataSource.lookup.template_index &&
			record[dataSource.template_header]) {
			template = record[dataSource.template_header];
		}

		let instance = templateEngine.createInstance({
			record: record,
			template: template
		});

		// create template instance for this record
		await instance.initialize();
		await instance.render();
		let file = await instance.createFile();

		// upload file
		if (settings.upload)
			file = await googleDrive.uploadFile(file);

		// save reference to file
		if (settings.change)
			dataSource.saveFileReference(index, file);

		await instance.cleanup();
	}
}

async function main() {
	// load settings
	console.log('load settings and command line arguments...');
	settings = configure();
	console.log('settings loaded:');
	console.log(settings);

	// load Google Client if necessary
	if (!settings.csv_file | settings.upload) {
		console.log('initialize OAuth client...');
		googleClient = new GoogleClient({
			credentials_file: settings.credentials_file,
			token_file: settings.token_file,
			scopes: SCOPES
		});
		await googleClient.initialize();
	}

	// load Google Drive if necessary
	if (!settings.csv_file | settings.upload) {
		console.log('initializing drive connection...');
		googleDrive = new GoogleDrive({
			upload: settings.upload,
			replace: settings.replace,
			google_client: googleClient,
			folder_id: settings.google_folder_id
		});
		await googleDrive.initialize();
	}

	// load data source
	console.log('initialize data source...');
	dataSource = new DataSource({
		template_header: settings.template_header,
		file_header: settings.file_header,
		change: settings.change,
		csv_file: settings.csv_file,
{'fraction_non_alphanumeric': 0.12644169158479282, 'fraction_numerical': 0.0022782286772034745, 'mean_word_length': 4.1609110947832475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 10, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3642055', 'n_tokens_mistral': 2171, 'n_tokens_neox': 1959, 'n_words': 586}