<commit_before>﻿using Dashboard.Models;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Table;
using Dashboard;
using Dashboard.Azure;
using Dashboard.Jenkins;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using Dashboard.Helpers;

namespace Dashboard.Controllers
{
    public class BuildsController : Controller
    {
        private readonly DashboardStorage _storage;

        public BuildsController()
        {
            var connectionString = CloudConfigurationManager.GetSetting(SharedConstants.StorageConnectionStringName);
            _storage = new DashboardStorage(connectionString);
        }

        /// <summary>
        /// Lists all of the build failures.
        /// </summary>
        public ActionResult Index(bool pr = false, DateTime? startDate = null, int limit = 10)
        {
            var startDateValue = _storage.GetStartDateValue(startDate);
            var failureQuery = _storage.GetBuildFailureEntities(startDateValue)
                .Where(x => pr || !JobUtil.IsPullRequestJobName(x.BuildId.JobName))
                .GroupBy(x => x.RowKey)
                .Select(x => new { Key = x.Key, Count = x.Count() })
                .OrderByDescending(x => x.Count)
                .Take(limit);

            var summary = new BuildFailureSummary()
            {
                IncludePullRequests = pr,
                StartDate = startDateValue,
                Limit = limit,
            };

            foreach (var pair in failureQuery)
            {
                var entry = new BuildFailureEntry()
                {
                    Name = pair.Key,
                    Count = pair.Count
                };
                summary.Entries.Add(entry);
            }

            return View(viewName: "FailureList", model: summary);
        }

        /// <summary>
        /// Summarize the details of an individual failure.
        /// </summary>
        public ActionResult Failure(string name = null, bool pr = true, DateTime? startDate = null)
        {
            var startDateValue = _storage.GetStartDateValue(startDate);
            var model = new BuildFailureModel()
            {
                Name = name,
                IncludePullRequests = pr,
                StartDate = startDateValue
            };

            foreach (var entity in _storage.GetBuildFailureEntities(name, startDateValue))
            {
                var buildId = entity.BuildId;
                if (!pr && JobUtil.IsPullRequestJobName(buildId.JobName))
                {
                    continue;
                }

                model.Builds.Add(entity);
            }

            return View(viewName: "Failure", model: model);
        }

        public ActionResult Demand(string userName, string commit)
        {
            var runStatus = new DemandRunStatusModel()
            {
                UserName = userName,
                Commit = commit,
            };

            var query = new TableQuery<DemandBuildEntity>()
                .Where(DashboardStorage.GenerateDemandBuildFilter(userName, commit));
            foreach (var entity in _storage.DemandBuildTable.ExecuteQuery(query))
            {
                var status = new DemandBuildStatusModel()
                {
                    BuildNumber = entity.BuildNumber,
                    JobName = entity.JobName,
                    QueueNumber = entity.QueueItemNumber
                };
                runStatus.StatusList.Add(status);
            }

            return View(viewName: "DemandStatus", model: runStatus);
        }
    }
}<commit_msg>CSV work<commit_after>﻿using Dashboard.Models;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Table;
using Dashboard;
using Dashboard.Azure;
using Dashboard.Jenkins;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Web;
using System.Web.Mvc;
using Dashboard.Helpers;
using System.Text;

namespace Dashboard.Controllers
{
    public class BuildsController : Controller
    {
        private readonly DashboardStorage _storage;

        public BuildsController()
        {
            var connectionString = CloudConfigurationManager.GetSetting(SharedConstants.StorageConnectionStringName);
            _storage = new DashboardStorage(connectionString);
        }

        /// <summary>
        /// Lists all of the build failures.
        /// </summary>
        public ActionResult Index(bool pr = false, DateTime? startDate = null, int limit = 10)
        {
            var summary = GetBuildFailureSummary(pr, startDate, limit);
            return View(viewName: "FailureList", model: summary);
        }

        /// <summary>
        /// Summarize the details of an individual failure.
        /// </summary>
        public ActionResult Failure(string name = null, bool pr = true, DateTime? startDate = null)
        {
            var startDateValue = _storage.GetStartDateValue(startDate);
            var model = new BuildFailureModel()
            {
                Name = name,
                IncludePullRequests = pr,
                StartDate = startDateValue
            };

            foreach (var entity in _storage.GetBuildFailureEntities(name, startDateValue))
            {
                var buildId = entity.BuildId;
                if (!pr && JobUtil.IsPullRequestJobName(buildId.JobName))
                {
                    continue;
                }

                model.Builds.Add(entity);
            }

            return View(viewName: "Failure", model: model);
        }

        public string Csv(bool pr = false, DateTime? startDate = null, int limit = 10)
        {
            var startDateValue = startDate ?? DateTime.UtcNow - TimeSpan.FromDays(7);
            var summary = GetBuildFailureSummary(pr, startDateValue, limit);
            var builder = new StringBuilder();
            foreach (var entry in summary.Entries)
            {
                var name = entry.Name.Replace(',', ' ');
                var index = name.LastIndexOf('.');
                var suiteName = name.Substring(0, index);
                var testName = name.Substring(index + 1);
                builder.AppendLine($"{suiteName},{testName},{entry.Count}");
            }

            return builder.ToString();
        }

        public ActionResult Demand(string userName, string commit)
        {
            var runStatus = new DemandRunStatusModel()
            {
                UserName = userName,
                Commit = commit,
            };

            var query = new TableQuery<DemandBuildEntity>()
                .Where(DashboardStorage.GenerateDemandBuildFilter(userName, commit));
            foreach (var entity in _storage.DemandBuildTable.ExecuteQuery(query))
            {
                var status = new DemandBuildStatusModel()
                {
                    BuildNumber = entity.BuildNumber,
                    JobName = entity.JobName,
                    QueueNumber = entity.QueueItemNumber
                };
                runStatus.StatusList.Add(status);
            }

            return View(viewName: "DemandStatus", model: runStatus);
        }

        public BuildFailureSummary GetBuildFailureSummary(bool pr = false, DateTime? startDate = null, int limit = 10)
        {
            var startDateValue = _storage.GetStartDateValue(startDate);
            var failureQuery = _storage.GetBuildFailureEntities(startDateValue)
                .Where(x => pr || !JobUtil.IsPullRequestJobName(x.BuildId.JobName))
                .GroupBy(x => x.RowKey)
                .Select(x => new { Key = x.Key, Count = x.Count() })
                .OrderByDescending(x => x.Count)
                .Take(limit);

            var summary = new BuildFailureSummary()
            {
                IncludePullRequests = pr,
                StartDate = startDateValue,
                Limit = limit,
            };

            foreach (var pair in failureQuery)
            {
                var entry = new BuildFailureEntry()
                {
                    Name = pair.Key,
                    Count = pair.Count
                };
                summary.Entries.Add(entry);
            }

            return summary;
        }
    }
}
{'fraction_non_alphanumeric': 0.08667299027744842, 'fraction_numerical': 0.001304244723737254, 'mean_word_length': 1.6583674755751654, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5272943', 'n_tokens_mistral': 2121, 'n_tokens_neox': 2014, 'n_words': 482}