<commit_before>      //Bottom of this doc sets up page structure and references components created above
      
      //Global variable for moro_click database
      var global_click_database = [];
      
      //These are imports from ReactRouter o.13.x
      //docs: https://github.com/rackt/react-router/blob/0.13.x/docs/guides/overview.md
      var Link = ReactRouter.Link;
      var RouteHandler = ReactRouter.RouteHandler;
      var Route = ReactRouter.Route;

      // These are endpoints to load data from.
      // Currently point to cloudant replication of lingsync data.
      // On cloudant we have a clean data view that doesn't exist on lingsync.

      // Endpoint for all up-to-date sentence data from all stories.
      var sentence_url = 'https://sande570.cloudant.com/psejenks-moro/_design/views_for_website/_view/clean_sentences';
      // Endpoint mapping story id to story name
      var story_url = 'https://sande570.cloudant.com/psejenks-moro/_design/views_for_website/_view/clean_stories';

      // Promise that is resolved once the sentence data is loaded
      var raw_data_promise = new Promise(function(resolve, reject) {
        $.ajax({
            url: sentence_url,
            dataType: 'json',
            cache: false,
            success: function(d) {
              resolve(d);
            },
            error: function(xhr, status, err) {
              console.error(sentence_url, status, err.toString());
                reject(err);}

          })
      });

      // Promise that is resolved once stories are loaded
      var story_data_promise = new Promise(function(resolve, reject) {
        $.ajax({
            url: story_url,
            dataType: 'json',
            cache: false,
            success: function(d) {
              resolve(d);
            },
            error: function(xhr, status, err) {
              console.error(sentence_url, status, err.toString());
                reject(err);}

          })
      });

      //===========================================Dictionary Code===========================================

      //get id of all occurrences of the morpheme and definition pair from the global_click_database
      function get_occurrence_ids(morpheme_click, definition_click) {
        var results = [];
        for (var i = 0; i < global_click_database.length; i++) {

            var morpheme_definition_pair = global_click_database[i]["morpheme_definition"];
                
            var match_found = false;
            for (var j = 0; j < morpheme_definition_pair.length; j++) {
                if (morpheme_definition_pair[j]["moroword"] == morpheme_click && morpheme_definition_pair[j]["definition"] == definition_click) {
                    match_found = true;
                    break;
                }
            }
            if (match_found) {
                results = results.concat (global_click_database[i]["id"]);
            //{sentence_id:dirtydata.rows[i].id, utterance_match:sentence.utterance, morphemes_match:sentence.morphemes, gloss_match:sentence.gloss, translation_match:sentence.translation});
            }
            
        }
        //console.log(results);
        return results;
      }


      //Segments a word into morphemes with glosses; morphemes from 'word' argument, glosses from 'glossword' argument
      function processword(word, glossword) {
        if (!word || !glossword) {
          return [[], []]
        }
        var results = [];
        var click_database_result = [];
        var morphemes = word.split('-');
        var glosses = glossword.split('-');
        //if there is not the same number of dashes we aren't aligning the correct morphemes and gloss
        if (morphemes.length!=glosses.length) {
          return [[], []];
        }
        var rootindex = -1;
        //identify verb roots so we can distinguish prefixes from suffixes
        for (var i = 0; i < glosses.length; i++) {
          var gloss = glosses[i];
          //all verb root morphemes end with .rt or .aux
          //TODO: does this include be.loc, be.1d, be.2d, etc? @HSande for details
          if (_.endsWith(gloss, '.rt') || _.endsWith(gloss, '.aux')) {
            rootindex = i;
          }
        }
        //iterate over morphemes; if there is a verb root, add pre-dashes to suffixes and post-dashes to prefixes: 
        //example: g-a-s-o; clg-rtc-eat.rt-pfv = [g-, a-, s, -o]; [clg-, rtc-, eat.rt, -pfv]
        for (var i = 0; i < glosses.length; i++) {
          var gloss = removePunc(glosses[i].toLowerCase());
          var morpheme = removePunc(morphemes[i].toLowerCase());
          if (rootindex==-1) {
            results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
            click_database_result.push({moroword:morpheme, definition:gloss});
          } else {
            if (i < rootindex) { 
             gloss = gloss+'-';
              morpheme = morpheme+'-';
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            } else if (i > rootindex) {
              gloss = '-'+gloss;
              morpheme = '-'+morpheme;
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            } else {
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            }
          }
        }
        return [results, click_database_result];
      }

    //merge two arrays and de-duplicate items
    function arrayUnique(array) {
        var a = array.concat();
        for(var i=0; i<a.length; ++i) {
            for(var j=i+1; j<a.length; ++j) {
                if(a[i]["word"] === a[j]["word"]) {
                    a.splice(j--, 1);
                    a[i]["count"] += 1 
                }
            }
        }

        return a;
    }

    //remove duplicate items for click morpheme_definition_pair_list
    function arrayUniqueClick(array) {
        var a = array.concat();
        for(var i=0; i<a.length; ++i) {
            for(var j=i+1; j<a.length; ++j) {
                if(a[i]["moroword"] === a[j]["moroword"] && a[i]["definition"] === a[j]["definition"]) {
                    a.splice(j--, 1);
                }
            }
        }

        return a;
    }

    //Remove punctuation from string excluding dashes and period in word
    function removePunc(word) {
        var rtnWord = word.replace(/[,\/#?!\"\“\”$%\^&\*;:{}=_`~()]/g,"");
        rtnWord = rtnWord.replace(/\b[.]+\B|\B[.]+\b/g, "");
        return rtnWord;
    }

    //Process dict with count to sorted dict without count value
    function sortAndRemoveCount(dict) {
        var toRtn = JSON.parse(JSON.stringify(dict));
        for(var i=0; i<toRtn.length; ++i) {
            toRtn[i]["moroword"].sort(function(a, b) {
                return parseFloat(b["count"]) - parseFloat(a["count"]);
            });
            var moroWordsArray = []
            for (var j=0; j<toRtn[i]["moroword"].length; ++j) {
                delete toRtn[i]["moroword"][j]["count"]
                var word = toRtn[i]["moroword"][j]["word"]
                moroWordsArray.push(word)
            }
            toRtn[i]["moroword"] = moroWordsArray
        }
        return toRtn
    }

      function processdata(dirtydata){
        var results = [];
        for (var i = 0; i < dirtydata.rows.length; i++) {
            // split on spaces and remove punctuation from morphemes line
            var sentence = dirtydata.rows[i].value.sentence; 
            var presplit_morphemes = sentence.morphemes.replace(/[",.?!'()]/g, '');
            var morphemes = presplit_morphemes.split(/[ ]/);
            var gloss = sentence.gloss.split(/[ ]/);
            
            var morpheme_definition_pair_list = []; //store morpheme definition pair of a sentence

            if (gloss.length = morphemes.length) {
                //process all morphemes and words
                for (var ii = 0; ii < gloss.length; ii++) {
                    var morpheme = morphemes[ii]; 
                    var glossword = gloss[ii];
                    var temp = processword(morpheme, glossword);
                    var wordresults = temp[0];
                    var click_database_results = temp[1];
                    var startIndex = 0;

                    morpheme_definition_pair_list = morpheme_definition_pair_list.concat(click_database_results); //add in the morpheme definition pair

                    if (results.length == 0) {
                        results = results.concat(wordresults[startIndex]);
                        startIndex += 1;
                    }
                    for (var k = startIndex; k < wordresults.length; k++) {
                        var existed = false;
                        for (var j = 0; j < results.length; j++) {
                            if (wordresults[k]["definition"] == results[j]["definition"]) {
                                existed = true;
                                oldMoroword = results[j]["moroword"];
                                newMoroword = arrayUnique(oldMoroword.concat(wordresults[k]["moroword"]));
                                results[j]["moroword"] = newMoroword;
                                break;
                            }       
                        }
                        if (!existed) {
                           results = results.concat(wordresults[k]);
                        }
                    }
                }
                //remove duplicate pair 
                morpheme_definition_pair_list = arrayUniqueClick(morpheme_definition_pair_list);
                //add the morpheme definition pair list for each sentence into the global variable
                global_click_database.push({id:dirtydata.rows[i].id, morpheme_definition:morpheme_definition_pair_list});
            }
        }
    //Print out result dict
    //console.log(JSON.stringify(results))
    //console.log(JSON.stringify(global_click_database))
    processedDict = sortAndRemoveCount(results)
    //console.log("DONE")
    //return morphemes/glosses by moro morphemes
    return _.sortBy(processedDict, function(j) {
      var moroword = _.cloneDeep(j.moroword);
      return _.map(moroword, function(x) {
        if (x[0] == '-') {
          return x.slice(1);
        }
        return x;
      });
    })
}

      // This is a test for processing code
      function assert(expected_value, actual) {
        if (!_.isEqual(expected_value, actual)){
          console.error('assertion failed');
          console.error(expected_value);
          console.error(JSON.stringify(expected_value));
          console.error(actual);
          console.error(JSON.stringify(actual));

        }
      }

      function test_processdata() {
        var testcase1 = {rows:[{value:{sentence:{morphemes:'a', gloss:'A'}}}]};
        assert([{moroword:['a'], definition:'a'}], processdata(testcase1));
        var testcase2 = {rows:[{value:{sentence: {morphemes:'a-b d', gloss:'A-B A'}}}]};
        assert([{moroword:['a','d'], definition:'a'}, {moroword:['b'], definition:'b'}], processdata(testcase2));
       var testcase3 = {rows:[{value:{sentence:{morphemes:'"loman-nǝŋ maj-anda l-a-fo,', gloss:'day-indef man-assoc.pl cll-rtc-past.aux'}}}]};
        assert([{moroword:['a-'], definition:'rtc-'},
                {moroword:['anda'], definition:'assoc.pl'},
                {moroword:['fo'], definition:'past.aux'},
                {moroword:['l-'], definition:'cll-'},
                {moroword:['loman'], definition:'day'}, 
                {moroword:['maj'], definition:'man'},
                {moroword:['nǝŋ'], definition:'indef'},
                 ], processdata(testcase3));
        var testcase4 = {rows:[{value:{sentence:{morphemes:'"a,!?..', gloss:'A'}}}]};
        assert([{moroword:['a'], definition:'a'}], processdata(testcase4));
        var testcase5 = {rows:[{value:{sentence:{morphemes:'b-a c', gloss:'B-A C'}}}]};
        assert([{moroword:['a'], definition:'a'}, {moroword:['b'], definition:'b'}, {moroword:['c'], definition:'c'}], processdata(testcase5));
        } 
      //test_processdata();

      // promise that resolves when sentence data is loaded and processed into morpheme dictionary
      var dictionary_data_promise = raw_data_promise.then(function(rawdata) {
        return processdata(rawdata); 
      });

      //Dictionary viewing code
      //ReactClass for rendering a definition
      var Definition = React.createClass({
        render: function() {
          var morph_def_pairs = _.map(this.props.moroword, function(morpheme) {
            return {
              morpheme: morpheme,
              definition: this.props.definition
            }
          }.bind(this));

          var rendered_morphemes = _.map(morph_def_pairs, function(pair, i) {
            var comma = ', ';
            if (i == 0) {
              comma = '';
            }
            var url = ('/#/dict/concordance/' + pair.morpheme + '/' +
                       pair.definition + '?' + CurrentMetaURI().query());
            return <span key={pair.morpheme}>
              {comma}
              <a href={url}>
                {pair.morpheme}
              </a>
            </span>;<commit_msg>relative hash URL so it works when run from non-root servers<commit_after>      //Bottom of this doc sets up page structure and references components created above
      
      //Global variable for moro_click database
      var global_click_database = [];
      
      //These are imports from ReactRouter o.13.x
      //docs: https://github.com/rackt/react-router/blob/0.13.x/docs/guides/overview.md
      var Link = ReactRouter.Link;
      var RouteHandler = ReactRouter.RouteHandler;
      var Route = ReactRouter.Route;

      // These are endpoints to load data from.
      // Currently point to cloudant replication of lingsync data.
      // On cloudant we have a clean data view that doesn't exist on lingsync.

      // Endpoint for all up-to-date sentence data from all stories.
      var sentence_url = 'https://sande570.cloudant.com/psejenks-moro/_design/views_for_website/_view/clean_sentences';
      // Endpoint mapping story id to story name
      var story_url = 'https://sande570.cloudant.com/psejenks-moro/_design/views_for_website/_view/clean_stories';

      // Promise that is resolved once the sentence data is loaded
      var raw_data_promise = new Promise(function(resolve, reject) {
        $.ajax({
            url: sentence_url,
            dataType: 'json',
            cache: false,
            success: function(d) {
              resolve(d);
            },
            error: function(xhr, status, err) {
              console.error(sentence_url, status, err.toString());
                reject(err);}

          })
      });

      // Promise that is resolved once stories are loaded
      var story_data_promise = new Promise(function(resolve, reject) {
        $.ajax({
            url: story_url,
            dataType: 'json',
            cache: false,
            success: function(d) {
              resolve(d);
            },
            error: function(xhr, status, err) {
              console.error(sentence_url, status, err.toString());
                reject(err);}

          })
      });

      //===========================================Dictionary Code===========================================

      //get id of all occurrences of the morpheme and definition pair from the global_click_database
      function get_occurrence_ids(morpheme_click, definition_click) {
        var results = [];
        for (var i = 0; i < global_click_database.length; i++) {

            var morpheme_definition_pair = global_click_database[i]["morpheme_definition"];
                
            var match_found = false;
            for (var j = 0; j < morpheme_definition_pair.length; j++) {
                if (morpheme_definition_pair[j]["moroword"] == morpheme_click && morpheme_definition_pair[j]["definition"] == definition_click) {
                    match_found = true;
                    break;
                }
            }
            if (match_found) {
                results = results.concat (global_click_database[i]["id"]);
            //{sentence_id:dirtydata.rows[i].id, utterance_match:sentence.utterance, morphemes_match:sentence.morphemes, gloss_match:sentence.gloss, translation_match:sentence.translation});
            }
            
        }
        //console.log(results);
        return results;
      }


      //Segments a word into morphemes with glosses; morphemes from 'word' argument, glosses from 'glossword' argument
      function processword(word, glossword) {
        if (!word || !glossword) {
          return [[], []]
        }
        var results = [];
        var click_database_result = [];
        var morphemes = word.split('-');
        var glosses = glossword.split('-');
        //if there is not the same number of dashes we aren't aligning the correct morphemes and gloss
        if (morphemes.length!=glosses.length) {
          return [[], []];
        }
        var rootindex = -1;
        //identify verb roots so we can distinguish prefixes from suffixes
        for (var i = 0; i < glosses.length; i++) {
          var gloss = glosses[i];
          //all verb root morphemes end with .rt or .aux
          //TODO: does this include be.loc, be.1d, be.2d, etc? @HSande for details
          if (_.endsWith(gloss, '.rt') || _.endsWith(gloss, '.aux')) {
            rootindex = i;
          }
        }
        //iterate over morphemes; if there is a verb root, add pre-dashes to suffixes and post-dashes to prefixes: 
        //example: g-a-s-o; clg-rtc-eat.rt-pfv = [g-, a-, s, -o]; [clg-, rtc-, eat.rt, -pfv]
        for (var i = 0; i < glosses.length; i++) {
          var gloss = removePunc(glosses[i].toLowerCase());
          var morpheme = removePunc(morphemes[i].toLowerCase());
          if (rootindex==-1) {
            results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
            click_database_result.push({moroword:morpheme, definition:gloss});
          } else {
            if (i < rootindex) { 
             gloss = gloss+'-';
              morpheme = morpheme+'-';
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            } else if (i > rootindex) {
              gloss = '-'+gloss;
              morpheme = '-'+morpheme;
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            } else {
              results.push({moroword:[{word:morpheme, count:1}], definition:gloss});
              click_database_result.push({moroword:morpheme, definition:gloss});
            }
          }
        }
        return [results, click_database_result];
      }

    //merge two arrays and de-duplicate items
    function arrayUnique(array) {
        var a = array.concat();
        for(var i=0; i<a.length; ++i) {
            for(var j=i+1; j<a.length; ++j) {
                if(a[i]["word"] === a[j]["word"]) {
                    a.splice(j--, 1);
                    a[i]["count"] += 1 
                }
            }
        }

        return a;
    }

    //remove duplicate items for click morpheme_definition_pair_list
    function arrayUniqueClick(array) {
        var a = array.concat();
        for(var i=0; i<a.length; ++i) {
            for(var j=i+1; j<a.length; ++j) {
                if(a[i]["moroword"] === a[j]["moroword"] && a[i]["definition"] === a[j]["definition"]) {
                    a.splice(j--, 1);
                }
            }
        }

        return a;
    }

    //Remove punctuation from string excluding dashes and period in word
    function removePunc(word) {
        var rtnWord = word.replace(/[,\/#?!\"\“\”$%\^&\*;:{}=_`~()]/g,"");
        rtnWord = rtnWord.replace(/\b[.]+\B|\B[.]+\b/g, "");
        return rtnWord;
    }

    //Process dict with count to sorted dict without count value
    function sortAndRemoveCount(dict) {
        var toRtn = JSON.parse(JSON.stringify(dict));
        for(var i=0; i<toRtn.length; ++i) {
            toRtn[i]["moroword"].sort(function(a, b) {
                return parseFloat(b["count"]) - parseFloat(a["count"]);
            });
            var moroWordsArray = []
            for (var j=0; j<toRtn[i]["moroword"].length; ++j) {
                delete toRtn[i]["moroword"][j]["count"]
                var word = toRtn[i]["moroword"][j]["word"]
                moroWordsArray.push(word)
            }
            toRtn[i]["moroword"] = moroWordsArray
        }
        return toRtn
    }

      function processdata(dirtydata){
        var results = [];
        for (var i = 0; i < dirtydata.rows.length; i++) {
            // split on spaces and remove punctuation from morphemes line
            var sentence = dirtydata.rows[i].value.sentence; 
            var presplit_morphemes = sentence.morphemes.replace(/[",.?!'()]/g, '');
            var morphemes = presplit_morphemes.split(/[ ]/);
            var gloss = sentence.gloss.split(/[ ]/);
            
            var morpheme_definition_pair_list = []; //store morpheme definition pair of a sentence

            if (gloss.length = morphemes.length) {
                //process all morphemes and words
                for (var ii = 0; ii < gloss.length; ii++) {
                    var morpheme = morphemes[ii]; 
                    var glossword = gloss[ii];
                    var temp = processword(morpheme, glossword);
                    var wordresults = temp[0];
                    var click_database_results = temp[1];
                    var startIndex = 0;

                    morpheme_definition_pair_list = morpheme_definition_pair_list.concat(click_database_results); //add in the morpheme definition pair

                    if (results.length == 0) {
                        results = results.concat(wordresults[startIndex]);
                        startIndex += 1;
                    }
                    for (var k = startIndex; k < wordresults.length; k++) {
                        var existed = false;
                        for (var j = 0; j < results.length; j++) {
                            if (wordresults[k]["definition"] == results[j]["definition"]) {
                                existed = true;
                                oldMoroword = results[j]["moroword"];
                                newMoroword = arrayUnique(oldMoroword.concat(wordresults[k]["moroword"]));
                                results[j]["moroword"] = newMoroword;
                                break;
                            }       
                        }
                        if (!existed) {
                           results = results.concat(wordresults[k]);
                        }
                    }
                }
                //remove duplicate pair 
                morpheme_definition_pair_list = arrayUniqueClick(morpheme_definition_pair_list);
                //add the morpheme definition pair list for each sentence into the global variable
                global_click_database.push({id:dirtydata.rows[i].id, morpheme_definition:morpheme_definition_pair_list});
            }
        }
    //Print out result dict
    //console.log(JSON.stringify(results))
    //console.log(JSON.stringify(global_click_database))
    processedDict = sortAndRemoveCount(results)
    //console.log("DONE")
    //return morphemes/glosses by moro morphemes
    return _.sortBy(processedDict, function(j) {
      var moroword = _.cloneDeep(j.moroword);
      return _.map(moroword, function(x) {
        if (x[0] == '-') {
          return x.slice(1);
        }
        return x;
      });
    })
}

      // This is a test for processing code
      function assert(expected_value, actual) {
        if (!_.isEqual(expected_value, actual)){
          console.error('assertion failed');
          console.error(expected_value);
          console.error(JSON.stringify(expected_value));
          console.error(actual);
          console.error(JSON.stringify(actual));

        }
      }

      function test_processdata() {
        var testcase1 = {rows:[{value:{sentence:{morphemes:'a', gloss:'A'}}}]};
        assert([{moroword:['a'], definition:'a'}], processdata(testcase1));
        var testcase2 = {rows:[{value:{sentence: {morphemes:'a-b d', gloss:'A-B A'}}}]};
        assert([{moroword:['a','d'], definition:'a'}, {moroword:['b'], definition:'b'}], processdata(testcase2));
       var testcase3 = {rows:[{value:{sentence:{morphemes:'"loman-nǝŋ maj-anda l-a-fo,', gloss:'day-indef man-assoc.pl cll-rtc-past.aux'}}}]};
        assert([{moroword:['a-'], definition:'rtc-'},
                {moroword:['anda'], definition:'assoc.pl'},
                {moroword:['fo'], definition:'past.aux'},
                {moroword:['l-'], definition:'cll-'},
                {moroword:['loman'], definition:'day'}, 
                {moroword:['maj'], definition:'man'},
                {moroword:['nǝŋ'], definition:'indef'},
                 ], processdata(testcase3));
        var testcase4 = {rows:[{value:{sentence:{morphemes:'"a,!?..', gloss:'A'}}}]};
        assert([{moroword:['a'], definition:'a'}], processdata(testcase4));
        var testcase5 = {rows:[{value:{sentence:{morphemes:'b-a c', gloss:'B-A C'}}}]};
        assert([{moroword:['a'], definition:'a'}, {moroword:['b'], definition:'b'}, {moroword:['c'], definition:'c'}], processdata(testcase5));
        } 
      //test_processdata();

      // promise that resolves when sentence data is loaded and processed into morpheme dictionary
      var dictionary_data_promise = raw_data_promise.then(function(rawdata) {
        return processdata(rawdata); 
      });

      //Dictionary viewing code
      //ReactClass for rendering a definition
      var Definition = React.createClass({
        render: function() {
          var morph_def_pairs = _.map(this.props.moroword, function(morpheme) {
            return {
              morpheme: morpheme,
              definition: this.props.definition
            }
          }.bind(this));

          var rendered_morphemes = _.map(morph_def_pairs, function(pair, i) {
            var comma = ', ';
            if (i == 0) {
              comma = '';
            }
            var url = ('#/dict/concordance/' + pair.morpheme + '/' +
                       pair.definition + '?' + CurrentMetaURI().query());
            return <span key={pair.morpheme}>
              {comma}
              <a href={url}>
                {pair.morpheme}
              </a>
            </span>;
{'fraction_non_alphanumeric': 0.14121656664443952, 'fraction_numerical': 0.003926798547825443, 'mean_word_length': 1.886857020639504, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4351723', 'n_tokens_mistral': 8181, 'n_tokens_neox': 7736, 'n_words': 1681}