<commit_before>﻿/*
Copyright (C) 2016 <NAME> <<EMAIL>>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Windows.Data.Json;
using Windows.Storage;
using System.IO;
using System.Text.RegularExpressions;

namespace AnkiU.AnkiCore.Importer
{
    public struct DuplicateNoteUpdate
    {        
        public bool isAllow;
        public bool isNotAskAgain;        
    }
    public class Anki2Importer : Importer
    {
        private const string DEFAULT_AFFIX = " Imported";

        private const int MEDIA_PICK_LIMIT= 1024;
        private const int MID = 2;

        private string deckPrefix;
        private DuplicateNoteUpdate isAllowUpdate;
        private bool dupeOnSchemaChange;

        private Dictionary<string, object[]> notes;

        /**
        * Java can't use a tuple as a key, so java ver resorts to indexing twice with nested maps.
        * Python: (guid, ord) -> cid
        * Java: guid -> ord -> cid
        * We will do the same in C# 
        */
        private Dictionary<string, Dictionary<int, long>> cards;
        private Dictionary<long, long> deckMapSourceToDest;
        private Dictionary<long, long> modelMap;
        private Dictionary<string, string> changedGuids;
        private Dictionary<string, bool> ignoredGuids;

        private int dupes;
        private int added;
        private int updated;       

        public int Dupes { get { return dupes; } }
        public int Added { get { return added; } }
        public int Updated { get { return updated; } }

        #region Not in java and python ver
        public delegate Task<DuplicateNoteUpdate> DuplicateNoteEventHandler();        
        public delegate Task<bool> DuplicateDeckEventhandler(string deckName);
        public delegate void ImporterStateHandler(string message);
        public event DuplicateNoteEventHandler DuplicateNoteEvent;
        public event DuplicateDeckEventhandler DuplicateDeckEvent;
        protected event ImporterStateHandler ImporterStateChangeEvent;

        private List<long> importedNoteId;
        public List<long> ImportedNoteId { get { return importedNoteId; } }

        private Dictionary<long, long> importedDeckIdMap;
        public Dictionary<long, long> ImportedDeckIdMap { get { return importedDeckIdMap; } }
        #endregion

        public Anki2Importer(Collection collection, StorageFolder sourceFolder, string relativePathToFile) 
            : base(collection, sourceFolder, relativePathToFile)
        {
            needMapper = false;
            deckPrefix = null;
            isAllowUpdate = new DuplicateNoteUpdate();    
            dupeOnSchemaChange = false;
            importedNoteId = new List<long>();
            importedDeckIdMap = new Dictionary<long, long>();
        }

        public async override Task Run()
        {
            await PrepareFiles();
            await Import();
        }

        private async Task PrepareFiles()
        {
            ImporterStateChangeEvent?.Invoke("Preparing collections...");
            sourceCol = await Storage.OpenOrCreateCollection(sourceFolder, relativePathToFile);
            deckMapSourceToDest = new Dictionary<long, long>();

            //WARNING: Not in java and python ver.                       
            await MakeSureNoConflictDeckName();
        }

        private async Task MakeSureNoConflictDeckName()
        {            
            var destDeckNames = destCol.Deck.AllNames();
            bool isDefaultDeck = false;
            foreach (var name in sourceCol.Deck.AllNames())
            {                
                string deckName = name;
                if (deckName.Equals(Constant.DEFAULT_DECKNAME, StringComparison.OrdinalIgnoreCase))
                {
                    deckName = ChangeDefaultDeckname(name, deckName);
                    isDefaultDeck = true;
                }

                if (destDeckNames.Contains(deckName))
                {
                    if (DuplicateDeckEvent == null)
                        continue;

                    bool isRename;
                    if (isDefaultDeck)
                        isRename = true;
                    else
                        isRename = await DuplicateDeckEvent(deckName);
                    if (isRename)
                    {                        
                        string newName = deckName + " (1)";
                        int index = 2;
                        while (destDeckNames.Contains(newName))
                        {
                            newName = deckName + " (" + index + ")";
                            index++;
                        }
                        var sourNames = sourceCol.Deck.AllNames();
                        while (sourNames.Contains(newName))
                        {
                            newName = deckName + " (" + index + ")";
                            index++;
                        }
                        RenameDeckInSource(deckName, newName);
                    }
                }
            }
        }

        private string ChangeDefaultDeckname(string name, string deckName)
        {
            var newName = deckName + DEFAULT_AFFIX;
            int index = 1;
            var sourNames = sourceCol.Deck.AllNames();
            while (sourNames.Contains(newName))
            {<commit_msg>Fix import sub-deck with same name<commit_after>﻿/*
Copyright (C) 2016 <NAME> <<EMAIL>>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Windows.Data.Json;
using Windows.Storage;
using System.IO;
using System.Text.RegularExpressions;

namespace AnkiU.AnkiCore.Importer
{
    public struct DuplicateNoteUpdate
    {        
        public bool isAllow;
        public bool isNotAskAgain;        
    }
    public class Anki2Importer : Importer
    {
        private const string DEFAULT_AFFIX = " Imported";

        private const int MEDIA_PICK_LIMIT= 1024;
        private const int MID = 2;

        private string deckPrefix;
        private DuplicateNoteUpdate isAllowUpdate;
        private bool dupeOnSchemaChange;

        private Dictionary<string, object[]> notes;

        /**
        * Java can't use a tuple as a key, so java ver resorts to indexing twice with nested maps.
        * Python: (guid, ord) -> cid
        * Java: guid -> ord -> cid
        * We will do the same in C# 
        */
        private Dictionary<string, Dictionary<int, long>> cards;
        private Dictionary<long, long> deckMapSourceToDest;
        private Dictionary<long, long> modelMap;
        private Dictionary<string, string> changedGuids;
        private Dictionary<string, bool> ignoredGuids;

        private int dupes;
        private int added;
        private int updated;       

        public int Dupes { get { return dupes; } }
        public int Added { get { return added; } }
        public int Updated { get { return updated; } }

        #region Not in java and python ver
        public delegate Task<DuplicateNoteUpdate> DuplicateNoteEventHandler();        
        public delegate Task<bool> DuplicateDeckEventhandler(string deckName);
        public delegate void ImporterStateHandler(string message);
        public event DuplicateNoteEventHandler DuplicateNoteEvent;
        public event DuplicateDeckEventhandler DuplicateDeckEvent;
        protected event ImporterStateHandler ImporterStateChangeEvent;

        private List<long> importedNoteId;
        public List<long> ImportedNoteId { get { return importedNoteId; } }

        private Dictionary<long, long> importedDeckIdMap;
        public Dictionary<long, long> ImportedDeckIdMap { get { return importedDeckIdMap; } }
        #endregion

        public Anki2Importer(Collection collection, StorageFolder sourceFolder, string relativePathToFile) 
            : base(collection, sourceFolder, relativePathToFile)
        {
            needMapper = false;
            deckPrefix = null;
            isAllowUpdate = new DuplicateNoteUpdate();    
            dupeOnSchemaChange = false;
            importedNoteId = new List<long>();
            importedDeckIdMap = new Dictionary<long, long>();
        }

        public async override Task Run()
        {
            await PrepareFiles();
            await Import();
        }

        private async Task PrepareFiles()
        {
            ImporterStateChangeEvent?.Invoke("Preparing collections...");
            sourceCol = await Storage.OpenOrCreateCollection(sourceFolder, relativePathToFile);
            deckMapSourceToDest = new Dictionary<long, long>();

            //WARNING: Not in java and python ver.                       
            await MakeSureNoConflictDeckName();
        }

        private async Task MakeSureNoConflictDeckName()
        {            
            var destDeckNames = destCol.Deck.AllNames();
            bool isDefaultDeck = false;           
            
            foreach (var deck in sourceCol.Deck.All())
            {                
                string deckName = deck.GetNamedString("name");
                if (deckName.Equals(Constant.DEFAULT_DECKNAME, StringComparison.OrdinalIgnoreCase))
                {
                    deckName = ChangeDefaultDeckname(deck.GetNamedString("name"), deckName);
                    isDefaultDeck = true;
                }

                if (destDeckNames.Contains(deckName))
                {
                    if (DuplicateDeckEvent == null)
                        continue;

                    bool isRename;
                    if (isDefaultDeck)
                    {
                        isRename = true;                        
                    }
                    else
                        isRename = await DuplicateDeckEvent(deckName);
                    if (isRename)
                    {                        
                        string newName = deckName + " (1)";
                        int index = 2;
                        while (destDeckNames.Contains(newName))
                        {
                            newName = deckName + " (" + index + ")";
                            index++;
                        }
                        var sourNames = sourceCol.Deck.AllNames();
                        while (sourNames.Contains(newName))
                        {
                            newName = deckName + " (" + index + ")";
                            index++;
                        }
                        RenameDeckInSource(deckName, newName);
                    }
                }

                isDefaultDeck = false;
            }
        }

        private string ChangeDefaultDeckname(string name, string deckName)
        {
            var newName = deckName + DEFAULT_AFFIX;
            int index = 1;
            var sourNames = sourceCol.Deck.AllNames();
            while (sourNames.Contains(newName))
            {
{'fraction_non_alphanumeric': 0.06892332789559544, 'fraction_numerical': 0.0024469820554649264, 'mean_word_length': 1.7374413931681179, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4427114', 'n_tokens_mistral': 3007, 'n_tokens_neox': 2867, 'n_words': 903}