<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using MyTy.Blog.Web.Models;
using MyTy.Blog.Web.Models.Github;
using RestSharp;

namespace MyTy.Blog.Web.Services
{
	public class GitHubMirror
	{
		readonly string gitHubBaseDir;
		readonly string localDir;		
		readonly GitHubAPI gitHubAPI;

		public GitHubMirror(string gitHubOwner, string gitHubRepo, string gitHubBaseDir, string localDir, string oAuthToken)
		{
			this.gitHubBaseDir = gitHubBaseDir;
			this.localDir = localDir;
			this.gitHubAPI = new GitHubAPI(gitHubOwner, gitHubRepo, "myty-blog-engine", oAuthToken);
		}

		public async Task<GitHubMirrorSynchronizeResult> SynchronizeAsync(bool okToDeleteFiles = true)
		{
			var result = new GitHubMirrorSynchronizeResult();

			try {

				var pagesContentResult = await gitHubAPI.GetContent(gitHubBaseDir);

				var treeContents = await Task.WhenAll(pagesContentResult.Where(p => p.type == "dir").Select(async p => {
					var treeResults = await gitHubAPI.GetTreeRecursively(p.sha);
					return new { 
						p.path,
						treeResults 
					};
				}));

				var filesPathSha = pagesContentResult.Where(p => p.type == "file").Select(f => new {
					path = f.path.Replace(gitHubBaseDir + "/", ""),
					sha = f.sha
				}).Concat(treeContents.SelectMany(t => t.treeResults.tree.Where(p => p.type == "blob").Select(p => new {
					path = t.path.Replace(gitHubBaseDir + "/", "") + "/" + p.path,
					sha = p.sha
				})));

				var remoteFilesSynced = (await Task.WhenAll(filesPathSha.Select(async f => {
					var getBlobTask = gitHubAPI.GetBlob(f.sha);

					var status = "";
					var localFileInfo = new System.IO.FileInfo(localDir + "\\" + f.path.Replace("/", "\\"));

					var blob = await getBlobTask;

					var fileContents = (blob.encoding == "base64") ?
						Encoding.UTF8.GetString(Convert.FromBase64String(blob.content)) :
						blob.content;


					if (!localFileInfo.Exists) {
						status = "added";
						localFileInfo.Directory.Create();
						File.WriteAllText(localFileInfo.FullName, fileContents);
					} else if (!File.ReadAllText(localFileInfo.FullName).Equals(fileContents)) {
						status = "updated";
						File.WriteAllText(localFileInfo.FullName, fileContents);
					}

					return new {
						localPath = localFileInfo.FullName,
						status
					};
				}))).ToArray();

				result.FilesUpdated = remoteFilesSynced
					.Where(f => f.status == "updated")
					.Select(f => f.localPath)
					.ToArray();

				result.FilesAdded = remoteFilesSynced
					.Where(f => f.status == "added")
					.Select(f => f.localPath)
					.ToArray();

				result.FilesDeleted = Directory
					.EnumerateFiles(localDir, "*", SearchOption.AllDirectories)
					.Where(f => !remoteFilesSynced.Any(s => s.localPath == f))
					.Select(f => {<commit_msg>bug fix: image files now sync correctly<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using System.Web;
using MyTy.Blog.Web.Models;
using MyTy.Blog.Web.Models.Github;
using RestSharp;

namespace MyTy.Blog.Web.Services
{
	public class GitHubMirror
	{
		readonly string gitHubBaseDir;
		readonly string localDir;		
		readonly GitHubAPI gitHubAPI;

		public GitHubMirror(string gitHubOwner, string gitHubRepo, string gitHubBaseDir, string localDir, string oAuthToken)
		{
			this.gitHubBaseDir = gitHubBaseDir;
			this.localDir = localDir;
			this.gitHubAPI = new GitHubAPI(gitHubOwner, gitHubRepo, "myty-blog-engine", oAuthToken);
		}

		public async Task<GitHubMirrorSynchronizeResult> SynchronizeAsync(bool okToDeleteFiles = true)
		{
			var result = new GitHubMirrorSynchronizeResult();

			try {

				var pagesContentResult = await gitHubAPI.GetContent(gitHubBaseDir);

				var treeContents = await Task.WhenAll(pagesContentResult.Where(p => p.type == "dir").Select(async p => {
					var treeResults = await gitHubAPI.GetTreeRecursively(p.sha);
					return new { 
						p.path,
						treeResults 
					};
				}));

				var filesPathSha = pagesContentResult.Where(p => p.type == "file").Select(f => new {
					path = f.path.Replace(gitHubBaseDir + "/", ""),
					sha = f.sha
				}).Concat(treeContents.SelectMany(t => t.treeResults.tree.Where(p => p.type == "blob").Select(p => new {
					path = t.path.Replace(gitHubBaseDir + "/", "") + "/" + p.path,
					sha = p.sha
				})));

				var remoteFilesSynced = (await Task.WhenAll(filesPathSha.Select(async f => {
					var getBlobTask = gitHubAPI.GetBlob(f.sha);

					string status = null;
					var localFileInfo = new System.IO.FileInfo(localDir + "\\" + f.path.Replace("/", "\\"));

					var blob = await getBlobTask;

					//var fileContents = (blob.encoding == "base64") ?
					//	Encoding.UTF8.GetString(Convert.FromBase64String(blob.content)) :
					//	blob.content;

					byte[] fileContents = Convert.FromBase64String(blob.content);
					FileStream fileStream = null;

					if (!localFileInfo.Exists) {
						status = "added";
						localFileInfo.Directory.Create();

						fileStream = new System.IO.FileStream(
							localFileInfo.FullName,
							System.IO.FileMode.Create,
							System.IO.FileAccess.Write);

						// File.WriteAllText(localFileInfo.FullName, fileContents);
					} else if (!File.ReadAllText(localFileInfo.FullName).Equals(fileContents)) {
						using (var md5 = MD5.Create()) {
							fileStream = new System.IO.FileStream(
								localFileInfo.FullName,
								System.IO.FileMode.Open,
								System.IO.FileAccess.ReadWrite);

							var currFileHash = md5.ComputeHash(fileStream);
							var newFileHash = md5.ComputeHash(fileContents);

							if (currFileHash.Length != newFileHash.Length) {
								status = "update";
							} else {
								for (int i = 0; i < currFileHash.Length; i++) {
									if (currFileHash[i] != newFileHash[i]) {
										status = "update";
										break;
									}
								}
							}
						}
					}

					if (status != null && fileStream != null) {
						fileStream.Position = 0L;
						fileStream.Write(fileContents, 0, fileContents.Length);
						fileStream.Close();
					}

					return new {
						localPath = localFileInfo.FullName,
						status
					};
				}))).ToArray();

				result.FilesUpdated = remoteFilesSynced
					.Where(f => f.status == "updated")
					.Select(f => f.localPath)
					.ToArray();

				result.FilesAdded = remoteFilesSynced
					.Where(f => f.status == "added")
					.Select(f => f.localPath)
					.ToArray();

				result.FilesDeleted = Directory
					.EnumerateFiles(localDir, "*", SearchOption.AllDirectories)
					.Where(f => !remoteFilesSynced.Any(s => s.localPath == f))
					.Select(f => {
{'fraction_non_alphanumeric': 0.13345169403757953, 'fraction_numerical': 0.002811066725847019, 'mean_word_length': 3.132029339853301, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3509008', 'n_tokens_mistral': 2792, 'n_tokens_neox': 2134, 'n_words': 413}