<commit_before>'use strict';


angular.module('arachne.visualizations.directives')
    .directive('con10tLineChart', ['$http', '$q', function ($http, $q) {
        return {
            restrict: 'E',
            scope: {
                letterDataPath: '@'
            },
            link: function (scope, element, attrs) {
                var dataQueries = [];

                dataQueries.push($http.get(scope.letterDataPath));

                scope.display = document.querySelector('#date-range-display-linechart');

                scope.selectionStartDate = null;
                scope.selectionEndDate = null;

                $q.all(dataQueries)
                    .then(function(responses) {
                        scope.processTsvData(
                            scope.parseTsvData(responses[0].data)
                        );
                        scope.initializeD3();
                        scope.updateState();
                    });

                scope.parseTsvData = function(data){
                    var parsedRows = [];
                    var lines = data.split('\n');

                    var headings = lines[0].split('\t');

                    var line_index = 1;
                    while (line_index < lines.length && lines[line_index].trim() !== '') {
                        var parsedLine = {};
                        var values = lines[line_index].split('\t');

                        var columnIndex = 0;

                        while (columnIndex < headings.length) {
                            parsedLine[headings[columnIndex]] = values[columnIndex];
                            columnIndex += 1;
                        }

                        parsedRows.push(parsedLine);
                        line_index += 1
                    }

                    return parsedRows
                };

                scope.processTsvData = function(tsvData) {
                    scope.data = [];
                    scope.binnedData = {};
                    scope.minDate = new Date('2018-01-01');
                    scope.maxDate = new Date('0000-01-01');

                    for(var i = 0; i < tsvData.length; i++){
                        var currentLetter = tsvData[i];

                        var fromDate = new Date(currentLetter['origin_date_from']);
                        var toDate = new Date(currentLetter['origin_date_to']);

                        if(fromDate.toString() === 'Invalid Date' || toDate.toString() === 'Invalid Date'){
                            continue;
                        }

                        if(fromDate.toISOString() === toDate.toISOString()) {
                            var binKey = fromDate.toISOString().substr(0,4);

                            if(new Date(binKey) < scope.minDate) scope.minDate = new Date(binKey);
                            if(new Date(binKey) > scope.maxDate) scope.maxDate = new Date(binKey);

                            if(binKey in scope.binnedData) {
                                scope.binnedData[binKey] += 1
                            } else {
                                scope.binnedData[binKey] = 1
                            }
                        } else {
                            var fromBinKey = fromDate.toISOString().substr(0,4);
                            var toBinKey = toDate.toISOString().substr(0,4);

                            if(new Date(fromBinKey) < scope.minDate) scope.minDate = new Date(fromBinKey);
                            if(new Date(toBinKey) > scope.maxDate) scope.maxDate = new Date(toBinKey);

                            if(fromBinKey in scope.binnedData) {
                                scope.binnedData[fromBinKey] += 1
                            } else {
                                scope.binnedData[fromBinKey] = 1
                            }

                            if(toBinKey in scope.binnedData) {
                                scope.binnedData[toBinKey] += 1
                            } else {
                                scope.binnedData[toBinKey] = 1
                            }
                        }
                    }

                    for(var binKey in scope.binnedData){
                        scope.data.push({
                            'date': new Date(binKey),
                            'count': scope.binnedData[binKey]
                        });
                    }

                    scope.data.sort(function(a, b) {
                        return a.date - b.date;
                    });
                };

                scope.initializeD3 = function(){
                    var margin = {top: 20, right: 50, bottom: 30, left: 50},
                        width = 960 - margin.left - margin.right,
                        height = 200 - margin.top - margin.bottom;

                    var bisectDate = d3.bisector(function(d) { return d.date; }).left;
<commit_msg>Fill years without data with 0 values<commit_after>'use strict';


angular.module('arachne.visualizations.directives')
    .directive('con10tLineChart', ['$http', '$q', function ($http, $q) {
        return {
            restrict: 'E',
            scope: {
                letterDataPath: '@'
            },
            link: function (scope, element, attrs) {
                var dataQueries = [];

                dataQueries.push($http.get(scope.letterDataPath));

                scope.display = document.querySelector('#date-range-display-linechart');

                scope.selectionStartDate = null;
                scope.selectionEndDate = null;

                $q.all(dataQueries)
                    .then(function(responses) {
                        scope.processTsvData(
                            scope.parseTsvData(responses[0].data)
                        );
                        scope.initializeD3();
                        scope.updateState();
                    });

                scope.parseTsvData = function(data){
                    var parsedRows = [];
                    var lines = data.split('\n');

                    var headings = lines[0].split('\t');

                    var line_index = 1;
                    while (line_index < lines.length && lines[line_index].trim() !== '') {
                        var parsedLine = {};
                        var values = lines[line_index].split('\t');

                        var columnIndex = 0;

                        while (columnIndex < headings.length) {
                            parsedLine[headings[columnIndex]] = values[columnIndex];
                            columnIndex += 1;
                        }

                        parsedRows.push(parsedLine);
                        line_index += 1
                    }

                    return parsedRows
                };

                scope.processTsvData = function(tsvData) {
                    scope.data = [];
                    scope.binnedData = {};
                    scope.minDate = new Date('2018-01-01');
                    scope.maxDate = new Date('0000-01-01');

                    for(var i = 0; i < tsvData.length; i++){
                        var currentLetter = tsvData[i];

                        var fromDate = new Date(currentLetter['origin_date_from']);
                        var toDate = new Date(currentLetter['origin_date_to']);

                        if(fromDate.toString() === 'Invalid Date' || toDate.toString() === 'Invalid Date'){
                            continue;
                        }

                        if(fromDate.toISOString() === toDate.toISOString()) {
                            var binKey = fromDate.toISOString().substr(0,4);

                            if(new Date(binKey) < scope.minDate) scope.minDate = new Date(binKey);
                            if(new Date(binKey) > scope.maxDate) scope.maxDate = new Date(binKey);

                            if(binKey in scope.binnedData) {
                                scope.binnedData[binKey] += 1
                            } else {
                                scope.binnedData[binKey] = 1
                            }
                        } else {
                            var fromBinKey = fromDate.toISOString().substr(0,4);
                            var toBinKey = toDate.toISOString().substr(0,4);

                            if(new Date(fromBinKey) < scope.minDate) scope.minDate = new Date(fromBinKey);
                            if(new Date(toBinKey) > scope.maxDate) scope.maxDate = new Date(toBinKey);

                            if(fromBinKey in scope.binnedData) {
                                scope.binnedData[fromBinKey] += 1
                            } else {
                                scope.binnedData[fromBinKey] = 1
                            }

                            if(toBinKey in scope.binnedData) {
                                scope.binnedData[toBinKey] += 1
                            } else {
                                scope.binnedData[toBinKey] = 1
                            }
                        }
                    }

                    for(var binKey in scope.binnedData){
                        scope.data.push({
                            'date': new Date(binKey),
                            'count': scope.binnedData[binKey]
                        });
                    }

                    scope.data.sort(function(a, b) {
                        return a.date - b.date;
                    });

                    var getYearsInbetween = function (startYear, endYear) {
                        var result = [];
                        var currentYear = startYear + 1;

                        while(currentYear < endYear){
                            result.push(
                                {
                                    'date': new Date(currentYear, 1, 1),
                                    'count': 0
                                });
                            currentYear += 1;
                        }

                        return result;
                    };

                    for(var i = 0; i < scope.data.length - 1; i++) {

                        if(scope.data[i]['date'].getFullYear() + 1 !== scope.data[i + 1]['date'].getFullYear()){
                            var inbetween = getYearsInbetween(
                                scope.data[i]['date'].getFullYear(),
                                scope.data[i + 1]['date'].getFullYear()
                            );

                            for(var j = 0; j < inbetween.length; j++){
                                scope.data.splice(i + 1 + j, 0, inbetween[j])
                            }
                        }
                    }
                };

                scope.initializeD3 = function(){
                    var margin = {top: 20, right: 50, bottom: 30, left: 50},
                        width = 960 - margin.left - margin.right,
                        height = 200 - margin.top - margin.bottom;

                    var bisectDate = d3.bisector(function(d) { return d.date; }).left;

{'fraction_non_alphanumeric': 0.10213648246642026, 'fraction_numerical': 0.010997926620391238, 'mean_word_length': 0.9374781697520084, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3858395', 'n_tokens_mistral': 2798, 'n_tokens_neox': 2521, 'n_words': 471}