<commit_before>import React from 'react';
import MuseumScene from './MuseumScene'
const $ = require('jquery');
require('./../styles.css');

class WikiPage extends React.Component {
  constructor(props) {
    super(props);
    let location = window.location.toString().split('/wiki/')[1];
    //could set vrMode state based on url
    this.state = {
      page: location,
      vrMode: false,
      infoLoaded: false,
      displayHtml: []
    };
    window.addEventListener('popstate', (event) => {
      let location = window.location.toString().split('/wiki/')[1];
      //could set vrMode state based on url
      this.setState({
        infoLoaded: false,
        page: location
      });
      this.getWikiInformation.call(this, this.state.page);
    });
  }

  componentWillMount() {
    this.getWikiInformation.call(this, this.state.page);
  }

  getWikiInformation (page, callback) {
    let url = `https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=text&page=${page}&callback=?`
    callback = callback || function(){return;};
    $.ajax({
        type: 'GET',
        url: url,
        contentType: 'application/json; charset=utf-8',
        async: false,
        dataType: 'json',
        success: (data, textStatus, jqXHR) => {
          // Parse Wiki data into discrete sections by topic
          if(data.error) {
            console.error(data.error.info);
            this.setState({
              rawResults: '<div>The requested page does not exist.</div>',
              infoLoaded: true
            });
          } else {
            let rawResults = $('<div id="rawResults"/>').append(data.parse.text["*"])[0];
            let rawResultsClone = $('<div id="rawResults"/>').append($(data.parse.text["*"]).clone());
            rawResultsClone.find('.mw-editsection, .portal').empty();
            let filteredResults = $(rawResults).children('p, h2, h3, table, ul, ol, .thumb');
            filteredResults = filteredResults.filter((child, element) => element.innerText.length > 0);

            const parsedHtmlSections = [];
            let contentEnded = false;
            let lastSection = undefined;
            let lastElement = undefined;
            for(var i = 0; i < filteredResults.length; i++) {
              let htmlSection = filteredResults[i];
              if($(htmlSection).children('#See_also, #References, #External_links').length) {
                contentEnded = true;
              }
              if(!contentEnded) {
                $(htmlSection).css('padding', '0px 10px');
                $(htmlSection).children('.mw-editsection').empty(); //Remove 'Edit' tags on titles
                $(htmlSection).children('.reference').empty();

                // Create new section...
                if($(htmlSection).is('h2') //if header
                  || ($(htmlSection).is('h3') && !$(lastElement).is('h2')) //or h3 starts section
                  || !lastSection) { //or no previous section

                  let newSection = $('<section />').append(htmlSection);
                  parsedHtmlSections.push(newSection);
                  lastSection = newSection;

                // Create image section...
                } else if($(htmlSection).is('.thumb')) {
                  let img = $(htmlSection).find('img');
                  let title = $(htmlSection).find('.thumbcaption')[0].innerText; // .innerHTML;
                  img.attr('title', title);
                  let newSection = $('<section />').append(img);
                  parsedHtmlSections.push(newSection);

                // Handle stray content...
                } else {
                  if(lastSection.html().length + htmlSection.outerHTML.length < 2000) {
                    $(lastSection).append(htmlSection)
                  } 

                  // Handle stray content that overflows text display...
                  // else {
                  //   let newSection = $('<section/>').append(htmlSection);
                  //   parsedHtmlSections.push(newSection);
                  //   lastSection = newSection
                  // }
                }
                lastElement = htmlSection;
              }
            }
            this.setState({
              rawResults: rawResultsClone,
              displayHtml: parsedHtmlSections,
              infoLoaded: true
            });
            callback.call(this, null);
          }
        },
        error: function (errorMessage) {
            console.error('Error retrieving from wikipedia:', errorMessage);
            callback.call(this, errorMessage);
        }
    });
  }

  handleSubmit(e) {
    e.preventDefault();
    window.location = '/wiki/' + e.target[0].value;<commit_msg>Remove all references (even deep level) and adjust length to be more accurate based on only text instead of html<commit_after>import React from 'react';
import MuseumScene from './MuseumScene'
const $ = require('jquery');
require('./../styles.css');

class WikiPage extends React.Component {
  constructor(props) {
    super(props);
    let location = window.location.toString().split('/wiki/')[1];
    //could set vrMode state based on url
    this.state = {
      page: location,
      vrMode: false,
      infoLoaded: false,
      displayHtml: []
    };
    window.addEventListener('popstate', (event) => {
      let location = window.location.toString().split('/wiki/')[1];
      //could set vrMode state based on url
      this.setState({
        infoLoaded: false,
        page: location
      });
      this.getWikiInformation.call(this, this.state.page);
    });
  }

  componentWillMount() {
    this.getWikiInformation.call(this, this.state.page);
  }

  getWikiInformation (page, callback) {
    let url = `https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=text&page=${page}&callback=?`
    callback = callback || function(){return;};
    $.ajax({
        type: 'GET',
        url: url,
        contentType: 'application/json; charset=utf-8',
        async: false,
        dataType: 'json',
        success: (data, textStatus, jqXHR) => {
          // Parse Wiki data into discrete sections by topic
          if(data.error) {
            console.error(data.error.info);
            this.setState({
              rawResults: '<div>The requested page does not exist.</div>',
              infoLoaded: true
            });
          } else {
            let rawResults = $('<div id="rawResults"/>').append(data.parse.text["*"])[0];
            let rawResultsClone = $('<div id="rawResults"/>').append($(data.parse.text["*"]).clone());
            rawResultsClone.find('.mw-editsection, .portal').empty();
            let filteredResults = $(rawResults).children('p, h2, h3, table, ul, ol, .thumb');
            filteredResults = filteredResults.filter((child, element) => element.innerText.length > 0);

            const parsedHtmlSections = [];
            let contentEnded = false;
            let lastSection = undefined;
            let lastElement = undefined;
            for(var i = 0; i < filteredResults.length; i++) {
              let htmlSection = filteredResults[i];
              if($(htmlSection).children('#See_also, #References, #External_links').length) {
                contentEnded = true;
              }
              if(!contentEnded) {
                $(htmlSection).css('padding', '0px 10px');
                $(htmlSection).find('.mw-editsection').empty(); //Remove 'Edit' tags on titles
                $(htmlSection).find('.reference').empty();

                // Create new section...
                if($(htmlSection).is('h2') //if header
                  || ($(htmlSection).is('h3') && !$(lastElement).is('h2')) //or h3 starts section
                  || !lastSection) { //or no previous section

                  let newSection = $('<section />').append(htmlSection);
                  parsedHtmlSections.push(newSection);
                  lastSection = newSection;

                // Create image section...
                } else if($(htmlSection).is('.thumb')) {
                  let img = $(htmlSection).find('img');
                  let title = $(htmlSection).find('.thumbcaption')[0].innerText; // .innerHTML;
                  img.attr('title', title);
                  let newSection = $('<section />').append(img);
                  parsedHtmlSections.push(newSection);

                // Handle stray content...
                } else {
                  if(lastSection.text().length + htmlSection.innerText.length < 1400) {
                    $(lastSection).append(htmlSection)
                  } 

                  // Handle stray content that overflows text display...
                  // else {
                  //   let newSection = $('<section/>').append(htmlSection);
                  //   parsedHtmlSections.push(newSection);
                  //   lastSection = newSection
                  // }
                }
                lastElement = htmlSection;
              }
            }
            this.setState({
              rawResults: rawResultsClone,
              displayHtml: parsedHtmlSections,
              infoLoaded: true
            });
            callback.call(this, null);
          }
        },
        error: function (errorMessage) {
            console.error('Error retrieving from wikipedia:', errorMessage);
            callback.call(this, errorMessage);
        }
    });
  }

  handleSubmit(e) {
    e.preventDefault();
    window.location = '/wiki/' + e.target[0].value;
{'fraction_non_alphanumeric': 0.14311307830119085, 'fraction_numerical': 0.004426177679418274, 'mean_word_length': 1.8143534994068802, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1765897', 'n_tokens_mistral': 2516, 'n_tokens_neox': 2418, 'n_words': 526}