<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Web;

namespace AppMetrics.DataModel
{
	public partial class DataSource
	{
		public IQueryable<Session> Sessions
		{
			get
			{
				string appKey;
				TimeSpan period;
				GetParams(out appKey, out period);

				var res = GetSessions(appKey, period);

				return res.AsQueryable();
			}
		}

		public IQueryable<Record> Records
		{
			get
			{
				string appKey;
				TimeSpan period;
				GetParams(out appKey, out period);

				var res = GetRecords(appKey, period);

				return res.AsQueryable();
			}
		}

		static void GetParams(out string appKey, out TimeSpan period)
		{
			var args = HttpContext.Current.Request.Params;

			appKey = args["appKey"];

			var periodText = args["period"];
			period = TimeSpan.Parse(periodText);
		}

		public static List<Session> GetSessions(string appKey, TimeSpan period)
		{
			var dataPath = Path.Combine(AppSettings.DataStoragePath, appKey);
			return GetSessionsFromPath(dataPath, period);
		}

		public static List<Session> GetSessionsFromPath(string dataPath, TimeSpan period)
		{
			var res = new List<Session>();

			var curTime = DateTime.UtcNow;

			if (Directory.Exists(dataPath))
			{
				foreach (var filePath in Directory.GetFiles(dataPath, "*.*.txt", SearchOption.AllDirectories))
				{
					if (filePath.EndsWith(Const.LogFileName, StringComparison.OrdinalIgnoreCase))
						continue;

					var fileName = Path.GetFileNameWithoutExtension(filePath);
					var nameParts = fileName.Split('.');
					var sessionId = nameParts.Last();

					var timeText = nameParts.First().Replace('_', ':');
					var sessionCreationTime = DateTime.ParseExact(timeText, "u", CultureInfo.InvariantCulture);
					if (curTime - sessionCreationTime > period)
						continue;

					var lastUpdateTime = File.GetLastWriteTime(filePath);

					var session = new Session
									{
										FileName = filePath,
										Id = sessionId,
										CreationTime = sessionCreationTime,
										LastUpdateTime = lastUpdateTime,
									};
					res.Add(session);
				}

				res.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime));
			}

			return res;
		}

		public static List<Record> GetRecords(string appKey, TimeSpan period)
		{
			var dataPath = Path.Combine(AppSettings.DataStoragePath, appKey);
			return GetRecordsFromPath(dataPath, period);
		}

		public static List<Record> GetRecordsFromPath(string dataPath, TimeSpan period)
		{
			var res = new List<Record>();

			var sessions = GetSessionsFromPath(dataPath, period);

			foreach (var session in sessions)
			{
				var tmp = GetRecordsFromSession(session);
				res.AddRange(tmp);
			}

			return res;
		}

		public static List<Record> GetRecordsFromSession(Session session)
		{
			var res = new List<Record>();

			var text = File.ReadAllText(session.FileName);
			var lines = text.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);

			foreach (var line in lines)
			{
				var fields = line.Split('\t');

				var record = new Record
								{
									SessionId = session.Id,
									Time = DateTime.Parse(fields[0]),
									Name = fields[1],
									Value = fields[2],
								};
				res.Add(record);
			}

			return res;
		}
	}
}<commit_msg>bugfix<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Web;

namespace AppMetrics.DataModel
{
	public partial class DataSource
	{
		public IQueryable<Session> Sessions
		{
			get
			{
				string appKey;
				TimeSpan period;
				GetParams(out appKey, out period);

				var res = GetSessions(appKey, period);

				return res.AsQueryable();
			}
		}

		public IQueryable<Record> Records
		{
			get
			{
				string appKey;
				TimeSpan period;
				GetParams(out appKey, out period);

				var res = GetRecords(appKey, period);

				return res.AsQueryable();
			}
		}

		static void GetParams(out string appKey, out TimeSpan period)
		{
			var args = HttpContext.Current.Request.Params;

			appKey = args["appKey"];

			var periodText = args["period"];
			period = TimeSpan.Parse(periodText);
		}

		public static List<Session> GetSessions(string appKey, TimeSpan period)
		{
			var dataPath = Path.Combine(AppSettings.DataStoragePath, appKey);
			return GetSessionsFromPath(dataPath, period);
		}

		public static List<Session> GetSessionsFromPath(string dataPath, TimeSpan period)
		{
			var res = new List<Session>();

			var curTime = DateTime.UtcNow;

			if (Directory.Exists(dataPath))
			{
				foreach (var filePath in Directory.GetFiles(dataPath, "*.*.txt", SearchOption.AllDirectories))
				{
					if (filePath.EndsWith(Const.LogFileName, StringComparison.OrdinalIgnoreCase))
						continue;

					var fileName = Path.GetFileNameWithoutExtension(filePath);
					var nameParts = fileName.Split('.');
					var sessionId = nameParts.Last();

					var timeText = nameParts.First().Replace('_', ':');
					var sessionCreationTime = DateTime.ParseExact(timeText, "u", CultureInfo.InvariantCulture);
					if (curTime - sessionCreationTime > period)
						continue;

					var lastUpdateTime = File.GetLastWriteTime(filePath);

					var session = new Session
									{
										FileName = filePath,
										Id = sessionId,
										CreationTime = sessionCreationTime,
										LastUpdateTime = lastUpdateTime,
									};
					res.Add(session);
				}

				res.Sort((x, y) => x.CreationTime.CompareTo(y.CreationTime));
			}

			return res;
		}

		public static List<Record> GetRecords(string appKey, TimeSpan period)
		{
			var dataPath = Path.Combine(AppSettings.DataStoragePath, appKey);
			return GetRecordsFromPath(dataPath, period);
		}

		public static List<Record> GetRecordsFromPath(string dataPath, TimeSpan period)
		{
			var res = new List<Record>();

			var sessions = GetSessionsFromPath(dataPath, period);

			foreach (var session in sessions)
			{
				var tmp = GetRecordsFromSession(session);
				res.AddRange(tmp);
			}

			return res;
		}

		public static List<Record> GetRecordsFromSession(Session session)
		{
			var res = new List<Record>();

			string text;
			using (var stream = new FileStream(session.FileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
			{
				using (var reader = new StreamReader(stream, Encoding.UTF8))
				{
					text = reader.ReadToEnd();
				}
			}
			var lines = text.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);

			foreach (var line in lines)
			{
				var fields = line.Split('\t');

				var record = new Record
								{
									SessionId = session.Id,
									Time = DateTime.Parse(fields[0]),
									Name = fields[1],
									Value = fields[2],
								};
				res.Add(record);
			}

			return res;
		}
	}
}
{'fraction_non_alphanumeric': 0.1055656806392822, 'fraction_numerical': 0.0009813542688910696, 'mean_word_length': 2.698289269051322, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '953019', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2138, 'n_words': 493}