<commit_before>/*jslint vars: true, plusplus: true, devel: true, nomen: true, indent: 4, maxerr: 50 */
/*global define */
define(function (require) {

    "use strict";

    var $               = require('jquery'),
        Underscore      = require('underscore'),
        Handlebars      = require('handlebars'),
        Backbone        = require('backbone'),
        Marionette      = require('marionette'),
        i18n            = require('i18n'),
//        sax             = require('sax'),
        tplImportDoc    = require('text!tpl/CopyOrImport.html'),
        projModel       = require('app/models/project'),
        bookModel       = require('app/models/book'),
        spModel         = require('app/models/sourcephrase'),
        chapModel       = require('app/models/chapter'),
        scrIDs          = require('utils/scrIDs'),
        USFM            = require('utils/usfm'),
        lines           = [],
        fileList        = [],
        fileCount       = 0,
        curFileIdx      = 0,

        // Helper method to import the selected file into the specified project.
        // This method has sub-methods for text, usfm, usx and xml (Adapt It document) file types.
        importFile = function (file, project) {
            var status = "";
            var reader = new FileReader();
            var i = 0;
            var name = "";
            var doc = null;
            // callback method for when the FileReader has finished loading in the file
            reader.onloadend = function (e) {
                var value = "",
                    scrID = null,
                    bookName = "",
                    chap = 0,
                    verse = 0,
                    s = "",
                    t = "",
                    index = 0,
                    markers = "",
                    orig = null,
                    prepuncts = "",
                    midpuncts = "",
                    follpuncts = "",
                    newSP = null,
                    chapter = null,
                    book = null,
                    books = new bookModel.BookCollection(),
                    chapters = new chapModel.ChapterCollection(),
                    sourcePhrases = new spModel.SourcePhraseCollection(),
                    arr = [],
                    bookID = "",
                    chapterID = "",
                    spID = "";
                ///
                // FILE TYPE READERS
                ///
                
                // Plain Text document
                // We assume these are just text with no markup,
                // in a single chapter (this could change if needed)
                var readTextDoc = function (contents) {
                    var re = /\s+/;
                    var newline = new RegExp('[\n\r\f\u2028\u2029]+', 'g');
                    var prepunct = "";
                    var follpunct = "";
                    var needsNewLine = false;
                    var sp = null;
                    console.log("Reading text file:" + file.name);
                    index = 1;
                    bookName = file.name;
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    book.trigger('change');
                    // (for now, just one chapter -- eventually we could chunk this out based on file size)
                    chapterID = Underscore.uniqueId();
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: bookName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', bookName);
                    }
                    // parse the text file and create the SourcePhrases
                    arr = contents.replace(newline, " <p> ").split(re); // insert special <p> for linefeeds, then split on whitespace
//                            arr = contents.split(re);
                    for (i = 0; i < arr.length; i++) {
                        // text becomes a new SourcePhrase, <p> and punctuation become markers / punct
                        if (arr[i] === "<p>") {
                            // newline -- make a note and keep going
                            needsNewLine = true;
                            continue;
                        }
                        if (arr[i].length === 0) {
                            continue; // blank entry -- skip
                        }
                        // punctuation -- have to check our punctuation pairs
//                                if (false) {
//                                    // is this before or after a space? <<<<<<<<<<<<<<<<
//                                    break;
//                                }
                        // if we got here, it's a regular SourcePhrase word. Create a new SP and add any LF / punctuation
                        spID = Underscore.uniqueId();
                        sp = new spModel.SourcePhrase({
                            spid: spID,
                            chapterid: chapterID,
                            markers: (needsNewLine === true) ? "\\p" : "",
                            orig: null,
                            prepuncts: "",
                            midpuncts: "",
                            follpuncts: "",
                            source: arr[i],
                            target: ""
                        });
                        index++;
                        sourcePhrases.add(sp);
                        sp.trigger('change');
                        needsNewLine = false;
                    }
                    // for non-scripture texts, there are no verses. Keep track of how far we are by using a 
                    // negative value for the # of SourcePhrases in the text.
                    chapter.set('versecount', -(index));
                    // Update the status string
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {
                        curStatus += "<br>";
                    }
                    $("#status2").html(curStatus + status);
                    curFileIdx++;
                    var newWidth = "width:" + (100 / fileCount * curFileIdx) + "%;";
                    $("#progress").attr("style", newWidth);
                    if (fileCount === curFileIdx) {
                        // last document -- display the OK button
                        $("#OK").removeAttr("disabled");
                    }
                };
                
                // Paratext USX document
                // These are XML-flavored markup files exported from Paratext
                var readUSXDoc = function (contents) {
                    var prepunct = "";
                    var follpunct = "";
                    var sp = null;
                    var xmlDoc = $.parseXML(contents);
                    var $xml = $(xmlDoc);
                    var chapterName = "";
                    // find the USFM ID of this book
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var verseCount = 0;
                    var verses = [];
                    var lastAdapted = 0;
                    var firstChapterID = "";
                    var i = 0;
                    console.log("Reading XML file:" + file.name);
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    // the book ID (e.g., "MAT") is in a singleton <book> element of the USX file
                    scrID = scrIDList.where({id: $($xml).find("book").attr("code")})[0];
                    if (scrID === null) {
                        console.log("No ID matching this document: " + $($xml).find("book").attr("code"));
                        return null;
                    }
                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = 1;
                    bookName = file.name.substr(0, file.name.indexOf("."));
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                };
                
                // Adapt It XML document
                // While XML is a general purpose document format, we're looking
                // specifically for Adapt It XML document files; other files
                // will be skipped (for now)
                var readXMLDoc = function (contents) {
                    var prepunct = "";
                    var re = /\s+/;
                    var follpunct = "";
                    var sp = null;
                    var xmlDoc = $.parseXML(contents);
                    var chapterName = "";
                    // find the USFM ID of this book
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var verseCount = 0;
                    var verses = [];
                    var lastAdapted = 0;
                    var firstChapterID = "";
                    var markers = "";
                    var i = 0;
                    console.log("Reading XML file:" + file.name);
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    // Starting at the SourcePhrases ( <S ...> ), look for the \id element
                    // in the markers. We'll test this against the canonical usfm markers to learn more about this document.
                    i = contents.indexOf("<S ");
                    index = contents.indexOf("\\id", i);
                    if (index === -1) {
                        // no ID found -- this is most likely not an AI xml document.
                        // We'll just return for now
                        console.log("No ID element found (is this an AI XML document?) -- exiting.");
                        return null;
                    }
                    // we've found the \id element in the markers -- to get the value, we have to work
                    // backwards until we find the nearest "s" attribute
                    // e.g., <S s="MAT" ...>.
                    index = contents.lastIndexOf("s=", index) + 3;
                    scrID = scrIDList.where({id: contents.substr(index, contents.indexOf("\"", index) - index)})[0];
                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = 1;
                    bookName = file.name.substr(0, file.name.indexOf("."));
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                    // create the sourcephrases
                    var $xml = $(xmlDoc);
                    var stridx = 0;
                    $($xml).find("S").each(function (i) {
                        // If this is a new chapter (starting for ch 2 -- chapter 1 is created above),
                        // create a new chapter object
                        markers = $(this).attr('m');
                        if (markers && markers.indexOf("\\c ") !== -1 && markers.indexOf("\\c 1 ") === -1) {
                            // update the last adapted for the previous chapter before closing it out
                            chapter.set('versecount', verseCount);
                            chapter.set('lastadapted', lastAdapted);
                            verses.push(verseCount); // add this chapter's verseCount to the array
                            verseCount = 0; // reset for the next chapter
                            lastAdapted = 0; // reset for the next chapter
                            stridx = markers.indexOf("\\c ") + 3;
                            chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: markers.substr(stridx, markers.indexOf(" ", stridx) - stridx)});
                            chapterID = Underscore.uniqueId();
                            // create the new chapter
                            chapter = new chapModel.Chapter({
                                chapterid: chapterID,
                                bookid: bookID,
                                projectid: project.get('id'),
                                name: chapterName,
                                lastadapted: 0,
                                versecount: 0
                            });
                            chapters.add(chapter);
                            chapter.trigger('change');
                            console.log(": " + $(this).attr('s') + ", " + chapterID);
                        }
                        if (markers && markers.indexOf("\\v ") !== -1) {
                            verseCount++;
                            // check this sourcephrase for a target - if there is one, consider this verse adapted
                            // (note that we're only checking the FIRST sp of each verse, not EVERY sp in the verse)
                            if ($(this).attr('t')) {
                                lastAdapted++;
                            }
                        }
                        // if there are filtered text items, insert them now
                        if ($(this).attr('fi')) {
                            markers = "";
                            $(this).attr('fi').split(re).forEach(function (elt, index, array) {
                                if (elt.indexOf("~FILTER") > -1) {
                                    // do nothing -- skip first and last elements
                                } else if (elt.indexOf("\\") === 0) {
                                    // starting marker
                                    markers += elt;
                                } else if (elt.indexOf("\\") > 0) {
                                    // ending marker - it's concatenated with the preceding token, no space
                                    // create a sourcephrase with the first part of the token, using the marker
                                    // from the end
                                    markers += elt.substr(elt.indexOf("\\"));
                                    spID = Underscore.uniqueId();
                                    sp = new spModel.SourcePhrase({
                                        spid: spID,
                                        chapterid: chapterID,
                                        markers: markers,
                                        orig: null,
                                        prepuncts: $(this).attr('pp'),
                                        midpuncts: "",
                                        follpuncts: $(this).attr('fp'),
                                        source: elt.substr(0, elt.indexOf("\\") - 1),
                                        target: ""
                                    });
                                    index++;
                                    sourcePhrases.add(sp);
                                    sp.trigger('change');
                                    markers = ""; // reset
                                } else {
                                    // regular token - add as a new sourcephrase
                                    spID = Underscore.uniqueId();
                                    sp = new spModel.SourcePhrase({
                                        spid: spID,
                                        chapterid: chapterID,
                                        markers: markers,
                                        orig: null,
                                        prepuncts: $(this).attr('pp'),
                                        midpuncts: "",
                                        follpuncts: $(this).attr('fp'),
                                        source: elt,
                                        target: ""
                                    });
                                    index++;
                                    sourcePhrases.add(sp);
                                    sp.trigger('change');
                                    markers = ""; // reset
                                }
                            });
                        }
                        // create the next sourcephrase
//                        console.log(i + ": " + $(this).attr('s') + ", " + chapterID);
                        spID = Underscore.uniqueId();
                        sp = new spModel.SourcePhrase({
                            spid: spID,
                            chapterid: chapterID,
                            markers: $(this).attr('m'),
                            orig: null,
                            prepuncts: $(this).attr('pp'),
                            midpuncts: "",
                            follpuncts: $(this).attr('fp'),
                            source: $(this).attr('k'), // source w/o punctuation
                            target: $(this).attr('t')
                        });
                        index++;
                        sourcePhrases.add(sp);
                        sp.trigger('change');
                    });
                    // update the verse count and last adapted verse for the book
                    if (verseCount > 0) {
                        verses.push(verseCount);
                    }
                    book.set('chapters', verses);
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {
                        curStatus += "<br>";
                    }
                    $("#status2").html(curStatus + status);
                    curFileIdx++;
                    var newWidth = "width:" + (100 / fileCount * curFileIdx) + "%;";
                    $("#progress").attr("style", newWidth);
                    if (fileCount === curFileIdx) {
                        // last document -- display the OK button
                        $("#OK").removeAttr("disabled");
                    }
                };
                
                // USFM document
                // This is the file format for Bibledit and Paratext
                // See http://paratext.org/about/usfm for format specification
                var readUSFMDoc = function (contents) {
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var chapterName = "";
                    var sp = null;
                    var re = /\s+/;
                    var markerList = new USFM.MarkerCollection();
                    var marker = null;

                    console.log("Reading USFM file:" + file.name);
                    // find the ID of this book
                    index = contents.indexOf("\\id");
                    if (index === -1) {
                        // no ID found -- just return
                        return null;
                    }
                    markerList.fetch({reset: true, data: {name: ""}});
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    scrID = scrIDList.where({id: contents.substr(index + 4, 3)})[0];
//                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = contents.indexOf("\\h ");
                    if (index > -1) {
                        // get the name from the usfm itself
                        bookName = contents.substr(index + 3, (contents.indexOf("\n", index) - (index + 3)));
                    }
                    // add a book and chapter
                    bookID = Underscore.uniqueId();
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: [] // arr
                    });
                    books.add(book);
                    // Note that we're adding chapter 1 before we reach the \c 1 marker in the file --
                    // Usually there's a fair amount of front matter before we reach the chapter itself;
                    // rather than creating a chapter 0 (which would throw off the search stuff), we'll
                    // just add the front matter to chapter 1.
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                    // build SourcePhrases
                    arr = contents.split(re);
                    for (i = 0; i < arr.length; i++) {
                        // text becomes a new SourcePhrase, <p> and punctuation become markers / punct
                        if (arr[i].indexOf("\\") === 0) {
                            // marker -- which one?
//                            if ((arr[i] === "\\v") || (arr[i] === "\\\c")) {
//                                // join with the next
//                            }
                        }
//                        if (arr[i] === "<p>") {
//                            // newline -- make a note and keep going
//                        }
//                        if (arr[i].length === 0) {
//                            continue; // blank entry -- skip
//                        }
                        // punctuation -- have to check our punctuation pairs
//                                if (false) {
//                                    // is this before or after a space? <<<<<<<<<<<<<<<<
//                                    break;
//                                }
                        // if we got here, it's a regular SourcePhrase word. Create a new SP and add any LF / punctuation
                        spID = Underscore.uniqueId();
                        sp = new spModel.SourcePhrase({
                            spid: spID,
                            chapterid: chapterID,
                            markers: "",
                            orig: null,
                            prepuncts: "",
                            midpuncts: "",
                            follpuncts: "",
                            source: arr[i],
                            target: ""
                        });
                        index++;
                        sourcePhrases.add(sp);
                        sp.trigger('change');
//                        needsNewLine = false;
                    }
                    
                    // done parsing -- update the status
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {<commit_msg>USFM import (not complete)<commit_after>/*jslint vars: true, plusplus: true, devel: true, nomen: true, indent: 4, maxerr: 50 */
/*global define */
define(function (require) {

    "use strict";

    var $               = require('jquery'),
        Underscore      = require('underscore'),
        Handlebars      = require('handlebars'),
        Backbone        = require('backbone'),
        Marionette      = require('marionette'),
        i18n            = require('i18n'),
//        sax             = require('sax'),
        tplImportDoc    = require('text!tpl/CopyOrImport.html'),
        projModel       = require('app/models/project'),
        bookModel       = require('app/models/book'),
        spModel         = require('app/models/sourcephrase'),
        chapModel       = require('app/models/chapter'),
        scrIDs          = require('utils/scrIDs'),
        USFM            = require('utils/usfm'),
        lines           = [],
        fileList        = [],
        fileCount       = 0,
        curFileIdx      = 0,

        // Helper method to import the selected file into the specified project.
        // This method has sub-methods for text, usfm, usx and xml (Adapt It document) file types.
        importFile = function (file, project) {
            var status = "";
            var reader = new FileReader();
            var i = 0;
            var name = "";
            var doc = null;
            // callback method for when the FileReader has finished loading in the file
            reader.onloadend = function (e) {
                var value = "",
                    scrID = null,
                    bookName = "",
                    chap = 0,
                    verse = 0,
                    s = "",
                    t = "",
                    index = 0,
                    markers = "",
                    orig = null,
                    prepuncts = "",
                    midpuncts = "",
                    follpuncts = "",
                    newSP = null,
                    chapter = null,
                    book = null,
                    books = new bookModel.BookCollection(),
                    chapters = new chapModel.ChapterCollection(),
                    sourcePhrases = new spModel.SourcePhraseCollection(),
                    arr = [],
                    bookID = "",
                    chapterID = "",
                    spID = "";
                ///
                // FILE TYPE READERS
                ///
                
                // Plain Text document
                // We assume these are just text with no markup,
                // in a single chapter (this could change if needed)
                var readTextDoc = function (contents) {
                    var re = /\s+/;
                    var newline = new RegExp('[\n\r\f\u2028\u2029]+', 'g');
                    var prepunct = "";
                    var follpunct = "";
                    var needsNewLine = false;
                    var sp = null;
                    console.log("Reading text file:" + file.name);
                    index = 1;
                    bookName = file.name;
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    book.trigger('change');
                    // (for now, just one chapter -- eventually we could chunk this out based on file size)
                    chapterID = Underscore.uniqueId();
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: bookName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', bookName);
                    }
                    // parse the text file and create the SourcePhrases
                    arr = contents.replace(newline, " <p> ").split(re); // insert special <p> for linefeeds, then split on whitespace
//                            arr = contents.split(re);
                    for (i = 0; i < arr.length; i++) {
                        // text becomes a new SourcePhrase, <p> and punctuation become markers / punct
                        if (arr[i] === "<p>") {
                            // newline -- make a note and keep going
                            needsNewLine = true;
                            continue;
                        }
                        if (arr[i].length === 0) {
                            continue; // blank entry -- skip
                        }
                        // punctuation -- have to check our punctuation pairs
//                                if (false) {
//                                    // is this before or after a space? <<<<<<<<<<<<<<<<
//                                    break;
//                                }
                        // if we got here, it's a regular SourcePhrase word. Create a new SP and add any LF / punctuation
                        spID = Underscore.uniqueId();
                        sp = new spModel.SourcePhrase({
                            spid: spID,
                            chapterid: chapterID,
                            markers: (needsNewLine === true) ? "\\p" : "",
                            orig: null,
                            prepuncts: "",
                            midpuncts: "",
                            follpuncts: "",
                            source: arr[i],
                            target: ""
                        });
                        index++;
                        sourcePhrases.add(sp);
                        sp.trigger('change');
                        needsNewLine = false;
                    }
                    // for non-scripture texts, there are no verses. Keep track of how far we are by using a 
                    // negative value for the # of SourcePhrases in the text.
                    chapter.set('versecount', -(index));
                    // Update the status string
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {
                        curStatus += "<br>";
                    }
                    $("#status2").html(curStatus + status);
                    curFileIdx++;
                    var newWidth = "width:" + (100 / fileCount * curFileIdx) + "%;";
                    $("#progress").attr("style", newWidth);
                    if (fileCount === curFileIdx) {
                        // last document -- display the OK button
                        $("#OK").removeAttr("disabled");
                    }
                };
                
                // Paratext USX document
                // These are XML-flavored markup files exported from Paratext
                var readUSXDoc = function (contents) {
                    var prepunct = "";
                    var follpunct = "";
                    var sp = null;
                    var xmlDoc = $.parseXML(contents);
                    var $xml = $(xmlDoc);
                    var chapterName = "";
                    // find the USFM ID of this book
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var verseCount = 0;
                    var verses = [];
                    var lastAdapted = 0;
                    var firstChapterID = "";
                    var i = 0;
                    console.log("Reading XML file:" + file.name);
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    // the book ID (e.g., "MAT") is in a singleton <book> element of the USX file
                    scrID = scrIDList.where({id: $($xml).find("book").attr("code")})[0];
                    if (scrID === null) {
                        console.log("No ID matching this document: " + $($xml).find("book").attr("code"));
                        return null;
                    }
                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = 1;
                    bookName = file.name.substr(0, file.name.indexOf("."));
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                };
                
                // Adapt It XML document
                // While XML is a general purpose document format, we're looking
                // specifically for Adapt It XML document files; other files
                // will be skipped (for now)
                var readXMLDoc = function (contents) {
                    var prepunct = "";
                    var re = /\s+/;
                    var follpunct = "";
                    var sp = null;
                    var xmlDoc = $.parseXML(contents);
                    var chapterName = "";
                    // find the USFM ID of this book
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var verseCount = 0;
                    var verses = [];
                    var lastAdapted = 0;
                    var firstChapterID = "";
                    var markers = "";
                    var i = 0;
                    console.log("Reading XML file:" + file.name);
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    // Starting at the SourcePhrases ( <S ...> ), look for the \id element
                    // in the markers. We'll test this against the canonical usfm markers to learn more about this document.
                    i = contents.indexOf("<S ");
                    index = contents.indexOf("\\id", i);
                    if (index === -1) {
                        // no ID found -- this is most likely not an AI xml document.
                        // We'll just return for now
                        console.log("No ID element found (is this an AI XML document?) -- exiting.");
                        return null;
                    }
                    // we've found the \id element in the markers -- to get the value, we have to work
                    // backwards until we find the nearest "s" attribute
                    // e.g., <S s="MAT" ...>.
                    index = contents.lastIndexOf("s=", index) + 3;
                    scrID = scrIDList.where({id: contents.substr(index, contents.indexOf("\"", index) - index)})[0];
                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = 1;
                    bookName = file.name.substr(0, file.name.indexOf("."));
                    bookID = Underscore.uniqueId();
                    // Create the book and chapter 
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: []
                    });
                    books.add(book);
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                    // create the sourcephrases
                    var $xml = $(xmlDoc);
                    var stridx = 0;
                    $($xml).find("S").each(function (i) {
                        // If this is a new chapter (starting for ch 2 -- chapter 1 is created above),
                        // create a new chapter object
                        markers = $(this).attr('m');
                        if (markers && markers.indexOf("\\c ") !== -1 && markers.indexOf("\\c 1 ") === -1) {
                            // update the last adapted for the previous chapter before closing it out
                            chapter.set('versecount', verseCount);
                            chapter.set('lastadapted', lastAdapted);
                            verses.push(verseCount); // add this chapter's verseCount to the array
                            verseCount = 0; // reset for the next chapter
                            lastAdapted = 0; // reset for the next chapter
                            stridx = markers.indexOf("\\c ") + 3;
                            chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: markers.substr(stridx, markers.indexOf(" ", stridx) - stridx)});
                            chapterID = Underscore.uniqueId();
                            // create the new chapter
                            chapter = new chapModel.Chapter({
                                chapterid: chapterID,
                                bookid: bookID,
                                projectid: project.get('id'),
                                name: chapterName,
                                lastadapted: 0,
                                versecount: 0
                            });
                            chapters.add(chapter);
                            chapter.trigger('change');
                            console.log(": " + $(this).attr('s') + ", " + chapterID);
                        }
                        if (markers && markers.indexOf("\\v ") !== -1) {
                            verseCount++;
                            // check this sourcephrase for a target - if there is one, consider this verse adapted
                            // (note that we're only checking the FIRST sp of each verse, not EVERY sp in the verse)
                            if ($(this).attr('t')) {
                                lastAdapted++;
                            }
                        }
                        // if there are filtered text items, insert them now
                        if ($(this).attr('fi')) {
                            markers = "";
                            $(this).attr('fi').split(re).forEach(function (elt, index, array) {
                                if (elt.indexOf("~FILTER") > -1) {
                                    // do nothing -- skip first and last elements
                                } else if (elt.indexOf("\\") === 0) {
                                    // starting marker
                                    markers += elt;
                                } else if (elt.indexOf("\\") > 0) {
                                    // ending marker - it's concatenated with the preceding token, no space
                                    // create a sourcephrase with the first part of the token, using the marker
                                    // from the end
                                    markers += elt.substr(elt.indexOf("\\"));
                                    spID = Underscore.uniqueId();
                                    sp = new spModel.SourcePhrase({
                                        spid: spID,
                                        chapterid: chapterID,
                                        markers: markers,
                                        orig: null,
                                        prepuncts: $(this).attr('pp'),
                                        midpuncts: "",
                                        follpuncts: $(this).attr('fp'),
                                        source: elt.substr(0, elt.indexOf("\\") - 1),
                                        target: ""
                                    });
                                    index++;
                                    sourcePhrases.add(sp);
                                    sp.trigger('change');
                                    markers = ""; // reset
                                } else {
                                    // regular token - add as a new sourcephrase
                                    spID = Underscore.uniqueId();
                                    sp = new spModel.SourcePhrase({
                                        spid: spID,
                                        chapterid: chapterID,
                                        markers: markers,
                                        orig: null,
                                        prepuncts: $(this).attr('pp'),
                                        midpuncts: "",
                                        follpuncts: $(this).attr('fp'),
                                        source: elt,
                                        target: ""
                                    });
                                    index++;
                                    sourcePhrases.add(sp);
                                    sp.trigger('change');
                                    markers = ""; // reset
                                }
                            });
                        }
                        // create the next sourcephrase
//                        console.log(i + ": " + $(this).attr('s') + ", " + chapterID);
                        spID = Underscore.uniqueId();
                        sp = new spModel.SourcePhrase({
                            spid: spID,
                            chapterid: chapterID,
                            markers: $(this).attr('m'),
                            orig: null,
                            prepuncts: $(this).attr('pp'),
                            midpuncts: "",
                            follpuncts: $(this).attr('fp'),
                            source: $(this).attr('k'), // source w/o punctuation
                            target: $(this).attr('t')
                        });
                        index++;
                        sourcePhrases.add(sp);
                        sp.trigger('change');
                    });
                    // update the verse count and last adapted verse for the book
                    if (verseCount > 0) {
                        verses.push(verseCount);
                    }
                    book.set('chapters', verses);
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {
                        curStatus += "<br>";
                    }
                    $("#status2").html(curStatus + status);
                    curFileIdx++;
                    var newWidth = "width:" + (100 / fileCount * curFileIdx) + "%;";
                    $("#progress").attr("style", newWidth);
                    if (fileCount === curFileIdx) {
                        // last document -- display the OK button
                        $("#OK").removeAttr("disabled");
                    }
                };
                
                // USFM document
                // This is the file format for Bibledit and Paratext
                // See http://paratext.org/about/usfm for format specification
                var readUSFMDoc = function (contents) {
                    var scrIDList = new scrIDs.ScrIDCollection();
                    var chapterName = "";
                    var sp = null;
                    var re = /\s+/;
                    var markerList = new USFM.MarkerCollection();
                    var marker = null;

                    console.log("Reading USFM file:" + file.name);
                    // find the ID of this book
                    index = contents.indexOf("\\id");
                    if (index === -1) {
                        // no ID found -- just return
                        return null;
                    }
                    markerList.fetch({reset: true, data: {name: ""}});
                    scrIDList.fetch({reset: true, data: {id: ""}});
                    scrID = scrIDList.where({id: contents.substr(index + 4, 3)})[0];
//                    arr = scrID.get('chapters');
                    books.fetch({reset: true, data: {name: ""}});
                    if (books.where({scrid: (scrID.get('id'))}).length > 0) {
                        // this book is already in the list -- just return
                        return null;
                    }
                    index = contents.indexOf("\\h ");
                    if (index > -1) {
                        // get the name from the usfm itself
                        bookName = contents.substr(index + 3, (contents.indexOf("\n", index) - (index + 3)));
                    }
                    // add a book and chapter
                    bookID = Underscore.uniqueId();
                    book = new bookModel.Book({
                        bookid: bookID,
                        projectid: project.get('id'),
                        scrid: scrID.get('id'),
                        name: bookName,
                        filename: file.name,
                        chapters: [] // arr
                    });
                    books.add(book);
                    // Note that we're adding chapter 1 before we reach the \c 1 marker in the file --
                    // Usually there's a fair amount of front matter before we reach the chapter itself;
                    // rather than creating a chapter 0 (which would throw off the search stuff), we'll
                    // just add the front matter to chapter 1.
                    chapterID = Underscore.uniqueId();
                    chapterName = i18n.t("view.lblChapterName", {bookName: bookName, chapterNumber: "1"});
                    chapter = new chapModel.Chapter({
                        chapterid: chapterID,
                        bookid: bookID,
                        projectid: project.get('id'),
                        name: chapterName,
                        lastadapted: 0,
                        versecount: 0
                    });
                    chapters.add(chapter);
                    // set the lastDocument / lastAdapted<xxx> values if not already set
                    if (project.get('lastDocument') === "") {
                        project.set('lastDocument', bookName);
                    }
                    if (project.get('lastAdaptedBookID') === 0) {
                        project.set('lastAdaptedBookID', bookID);
                        project.set('lastAdaptedChapterID', chapterID);
                    }
                    if (project.get('lastAdaptedName') === "") {
                        project.set('lastAdaptedName', chapterName);
                    }
                    // build SourcePhrases
                    arr = contents.split(re);
                    i = 0;
                    while (i < arr.length) {
                        // check for a marker
                        if (arr[i].indexOf("\\") === 0) {
                            markers += arr[i];
                            // Check for markers with more than one token (and merge the two marker tokens)
                            if ((arr[i] === "\\x") || (arr[i] === "\\f") ||
                                    (arr[i] === "\\c") || (arr[i] === "\\ca") || (arr[i] === "\\cp") ||
                                    (arr[i] === "\\v") || (arr[i] === "\\va") || (arr[i] === "\\vp")) {
                                // join with the next
                                i++;
                                markers += arr[i];
                            }
                        } else if (arr[i].length === 0) {
                            i++;
                            continue; // blank entry -- skip
                        } else {
                        // punctuation -- have to check our punctuation pairs
//                                if (false) {
//                                    // is this before or after a space? <<<<<<<<<<<<<<<<
//                                    break;
//                                }
                            // if we got here, it's a regular SourcePhrase word. Create a new SP and add any LF / punctuation
                            spID = Underscore.uniqueId();
                            sp = new spModel.SourcePhrase({
                                spid: spID,
                                chapterid: chapterID,
                                markers: "",
                                orig: null,
                                prepuncts: "",
                                midpuncts: "",
                                follpuncts: "",
                                source: arr[i],
                                target: ""
                            });
                            index++;
                            sourcePhrases.add(sp);
                            sp.trigger('change');
                            i++;
                        }
//                        needsNewLine = false;
                    }
                    
                    // done parsing -- update the status
                    if (status.length > 0) {
                        status += "<br>";
                    }
                    status += i18n.t("view.dscCopyDocumentFound", {document: bookName});
                    // update the status
                    var curStatus = $("#status2").html();
                    if (curStatus.length > 0) {
{'fraction_non_alphanumeric': 0.10445293054655802, 'fraction_numerical': 0.004057417573050564, 'mean_word_length': 0.8558863542886069, 'pattern_counts': {'":': 4, '<': 107, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 118, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4486847', 'n_tokens_mistral': 13459, 'n_tokens_neox': 12226, 'n_words': 3395}