<commit_before>'use strict';

angular.module('app').controller('rgiNewRefDialogCtrl', function (
    $scope,
    $route,
    $timeout,
    FileUploader,
    ngDialog,
    rgiNotifier,
    rgiAnswerMethodSrvc,
    rgiRequestSubmitterSrvc
) {
    $scope.answer_update = $scope.$parent.answer;
    ////TODO REPLACE WITH EXISITING REFERENCE SET
    //$scope.existing_ref = [
    //    {text: 'Add Document', value: 'document'},
    //    {text: 'Add Webpage', value: 'webpage'},
    //    {text: 'Add Interview', value: 'interview'}
    //];

    function isURLReal(fullyQualifiedURL) {
        var URL = encodeURIComponent(fullyQualifiedURL),
            dfd = $.Deferred(),
            checkURLPromise = $.getJSON('http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22' + URL + '%22&format=json');

        checkURLPromise
            .done(function(res) {
                // results should be null if the page 404s or the domain doesn't work
                if (res.query.results) {
                    dfd.resolve(true);
                } else {
                    dfd.reject(false);
                }
            })
            .fail(function () {
                dfd.reject('failed');
            });
        return dfd.promise();
    }

    $scope.fileUrl = '';

    $scope.uploadFileByUrl = function() {
        var handleFileUploadStatus = function(response) {
            $scope.uploader.queue[$scope.uploader.queue.length - 1].progress = response.data.completion * 100;

            if(response.data.completion < 1) {
                $timeout(function() {
                    rgiRequestSubmitterSrvc.get('/api/remote-file/upload-progress/' + response.data._id).then(handleFileUploadStatus);
                }, 1000);
            } else {
                uploader.onCompleteItem({}, {}, null);
            }
        };

        isURLReal($scope.fileUrl)
            .fail(function () {
                rgiNotifier.error('The remote file is not found');
            })
            .done(function () {
                rgiRequestSubmitterSrvc.get('/api/remote-file-upload?url=' + encodeURIComponent($scope.fileUrl)).then(function(response) {
                    $scope.uploader.queue.push({
                        file: {
                            name: $scope.fileUrl.split('/')[$scope.fileUrl.split('/').length - 1],
                            size: response.data.size
                        },
                        isUploading: true,
                        progress: response.data.completion * 100
                    });
                    handleFileUploadStatus(response);
                });
            });
    };

    //DATEPICKER OPTS
    $scope.date_format = 'MMMM d, yyyy';
    var today = new Date();
    $scope.date_default = today;
    $scope.date_max_limit = today;

    var uploader = $scope.uploader = new FileUploader({
        isHTML5: true,
        withCredentials: true,
        url: 'file-upload'
    });
    uploader.filters.push({
        name: 'customFilter',
        fn: function () {
            return this.queue.length < 1;
        }
    });

    uploader.onCompleteItem = function (fileItem, response, status) {
        if (status === 400) {
            $scope.uploader.queue = [];<commit_msg>validate remote files on the server<commit_after>'use strict';

angular.module('app').controller('rgiNewRefDialogCtrl', function (
    $scope,
    $route,
    $timeout,
    FileUploader,
    ngDialog,
    rgiNotifier,
    rgiAnswerMethodSrvc,
    rgiRequestSubmitterSrvc
) {
    $scope.answer_update = $scope.$parent.answer;
    ////TODO REPLACE WITH EXISITING REFERENCE SET
    //$scope.existing_ref = [
    //    {text: 'Add Document', value: 'document'},
    //    {text: 'Add Webpage', value: 'webpage'},
    //    {text: 'Add Interview', value: 'interview'}
    //];

    function isURLReal(fullyQualifiedURL) {
        var URL = encodeURIComponent(fullyQualifiedURL),
            dfd = $.Deferred(),
            checkURLPromise = $.getJSON('http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D%22' + URL + '%22&format=json');

        checkURLPromise
            .done(function(res) {
                // results should be null if the page 404s or the domain doesn't work
                if (res.query.results) {
                    dfd.resolve(true);
                } else {
                    dfd.reject(false);
                }
            })
            .fail(function () {
                dfd.reject('failed');
            });
        return dfd.promise();
    }

    $scope.fileUrl = '';

    $scope.uploadFileByUrl = function() {
        var handleFileUploadStatus = function(response) {
            $scope.uploader.queue[$scope.uploader.queue.length - 1].progress = response.data.completion * 100;

            if(response.data.completion < 1) {
                $timeout(function() {
                    rgiRequestSubmitterSrvc.get('/api/remote-file/upload-progress/' + response.data._id).then(handleFileUploadStatus);
                }, 1000);
            } else {
                uploader.onCompleteItem({}, {}, null);
            }
        };

        rgiRequestSubmitterSrvc.get('/api/remote-file-upload?url=' + encodeURIComponent($scope.fileUrl)).then(function(response) {
            if(response.data.reason) {
                rgiNotifier.error('The file cannot be uploaded');
            } else {
                $scope.uploader.queue.push({
                    file: {
                        name: $scope.fileUrl.split('/')[$scope.fileUrl.split('/').length - 1],
                        size: response.data.size
                    },
                    isUploading: true,
                    progress: response.data.completion * 100
                });
                handleFileUploadStatus(response);
            }
        });
    };

    //DATEPICKER OPTS
    $scope.date_format = 'MMMM d, yyyy';
    var today = new Date();
    $scope.date_default = today;
    $scope.date_max_limit = today;

    var uploader = $scope.uploader = new FileUploader({
        isHTML5: true,
        withCredentials: true,
        url: 'file-upload'
    });
    uploader.filters.push({
        name: 'customFilter',
        fn: function () {
            return this.queue.length < 1;
        }
    });

    uploader.onCompleteItem = function (fileItem, response, status) {
        if (status === 400) {
            $scope.uploader.queue = [];
{'fraction_non_alphanumeric': 0.1324349442379182, 'fraction_numerical': 0.011462205700123915, 'mean_word_length': 1.7256226255804137, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3820376', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1802, 'n_words': 316}