<commit_before>// @Author: <NAME>
// @Date: 12/8/2014 (created)
// @Description: Easy way to create shapefiles (and geojson, geoservices json)
//               from ArcGIS Server services
// @services.txt format :: serviceLayerURL|layerName
// @githubURL : https://github.com/tannerjt/AGStoShapefile

// Node Modules
var ogr2ogr = require('ogr2ogr');
var esri2geo = require('esri2geo');
var q = require('q');
var request = q.nfbind(require('request'));
var objectstream = require('objectstream');
var fs = require('fs');
var queryString = require('query-string');
var winston = require('winston');

// Setup logging with winston
winston.level = 'debug';
// winston.add(winston.transports.File, {filename: './logfile.log'});

// ./mixin.js
// merge user query params with default
var mixin = require('./mixin');

var serviceFile = process.argv[2] || 'services.txt';
var outDir = process.argv[3] || './output/';
if(outDir[outDir.length - 1] !== '/') {
	outDir += '/';
}

// Make request to each service
fs.readFile(serviceFile, function (err, data) {
	if (err) {
		winston.info(err);
		throw err;
	}
	data.toString().split('\n').forEach(function (service) {
		var service = service.split('|');
		if(service[0].split('').length == 0) return;
		var baseUrl = getBaseUrl(service[0].trim()) + '/query';

		var reqQS = {
			where: '1=1',
			returnIdsOnly: true,
			f: 'json'
		};
		var userQS = getUrlVars(service[0].trim());
		// mix one obj with another
		var qs = mixin(userQS, reqQS);
		var qs = queryString.stringify(qs);
		var url = decodeURIComponent(getBaseUrl(baseUrl) + '/query/?' + qs);

		request({
			url : url,
			method : 'GET',
			json : true
		}, function (err, response, body) {
			var err = err || body.error;
			if(err) {
				winston.info(err);
				throw err;
			}
			requestService(service[0].trim(), service[1].trim(), body.objectIds);
		});
	})
});

// Resquest JSON from AGS
function requestService(serviceUrl, serviceName, objectIds) {
	objectIds.sort();
	winston.info('Number of features for service: ', objectIds.length);
	winston.info('Getting chunks of 100 features...');
	var requests = [];

	for(var i = 0; i < Math.ceil(objectIds.length / 100); i++) {
		var ids = [];
		if ( ((i + 1) * 100) < objectIds.length ) {
			ids = objectIds.slice(i * 100, (i * 100) + 100);
		} else {
			ids = objectIds.slice(i * 100, objectIds[objectIds.length-1]);
		}

		if(ids[0] !== undefined) {
			winston.info('query ->', (i * 100) , 'out of', objectIds.length);
		} else {
			winston.info('wait for requests to settle...');
			continue;
		}

		// we need these query params
		var reqQS = {
			objectIds : ids.join(','),
			geometryType : 'esriGeometryEnvelope',
			returnGeometry : true,
			returnIdsOnly : false,
			outFields : '*',
			outSR : '4326',
			f : 'json'
		};
		// user provided query params
		var userQS = getUrlVars(serviceUrl);
		// mix one obj with another
		var qs = mixin(userQS, reqQS);
		var qs = queryString.stringify(qs);
		var url = decodeURIComponent(getBaseUrl(serviceUrl) + '/query/?' + qs);
		var r = request({
			url : url,
			method : 'GET',<commit_msg>Update logic based on issue #9 input<commit_after>// @Author: <NAME>
// @Date: 12/8/2014 (created)
// @Description: Easy way to create shapefiles (and geojson, geoservices json)
//               from ArcGIS Server services
// @services.txt format :: serviceLayerURL|layerName
// @githubURL : https://github.com/tannerjt/AGStoShapefile

// Node Modules
var ogr2ogr = require('ogr2ogr');
var esri2geo = require('esri2geo');
var q = require('q');
var request = q.nfbind(require('request'));
var objectstream = require('objectstream');
var fs = require('fs');
var queryString = require('query-string');
var winston = require('winston');

// Setup logging with winston
winston.level = 'debug';
// winston.add(winston.transports.File, {filename: './logfile.log'});

// ./mixin.js
// merge user query params with default
var mixin = require('./mixin');

var serviceFile = process.argv[2] || 'services.txt';
var outDir = process.argv[3] || './output/';
if(outDir[outDir.length - 1] !== '/') {
	outDir += '/';
}

// Make request to each service
fs.readFile(serviceFile, function (err, data) {
	if (err) {
		winston.info(err);
		throw err;
	}
	data.toString().split('\n').forEach(function (service) {
		var service = service.split('|');
		if(service[0].split('').length == 0) return;
		var baseUrl = getBaseUrl(service[0].trim()) + '/query';

		var reqQS = {
			where: '1=1',
			returnIdsOnly: true,
			f: 'json'
		};
		var userQS = getUrlVars(service[0].trim());
		// mix one obj with another
		var qs = mixin(userQS, reqQS);
		var qs = queryString.stringify(qs);
		var url = decodeURIComponent(getBaseUrl(baseUrl) + '/query/?' + qs);

		request({
			url : url,
			method : 'GET',
			json : true
		}, function (err, response, body) {
			var err = err || body.error;
			if(err) {
				winston.info(err);
				throw err;
			}
			requestService(service[0].trim(), service[1].trim(), body.objectIds);
		});
	})
});

// Resquest JSON from AGS
function requestService(serviceUrl, serviceName, objectIds) {
	objectIds.sort();
	winston.info('Number of features for service: ', objectIds.length);
	winston.info('Getting chunks of 100 features...');
	var requests = [];

	for(var i = 0; i < Math.ceil(objectIds.length / 100); i++) {
		var ids = [];
		if ( ((i + 1) * 100) < objectIds.length ) {
			ids = objectIds.slice(i * 100, (i * 100) + 100);
		} else {
			ids = objectIds.slice(i * 100, objectIds[objectIds.length]);
		}

		if(ids[0] !== undefined) {
			winston.info('query ->', (i * 100) , 'out of', objectIds.length);
		} else {
			winston.info('wait for requests to settle...');
			continue;
		}

		// we need these query params
		var reqQS = {
			objectIds : ids.join(','),
			geometryType : 'esriGeometryEnvelope',
			returnGeometry : true,
			returnIdsOnly : false,
			outFields : '*',
			outSR : '4326',
			f : 'json'
		};
		// user provided query params
		var userQS = getUrlVars(serviceUrl);
		// mix one obj with another
		var qs = mixin(userQS, reqQS);
		var qs = queryString.stringify(qs);
		var url = decodeURIComponent(getBaseUrl(serviceUrl) + '/query/?' + qs);
		var r = request({
			url : url,
			method : 'GET',
{'fraction_non_alphanumeric': 0.17572142511687894, 'fraction_numerical': 0.0174109301950669, 'mean_word_length': 3.5350877192982457, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4754497', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2121, 'n_words': 545}