<commit_before>/**
 * @module src/UploadTokens
 *
 * Component that display SourceDocument
 * Accepts the following properties:
 *  - language: Ethnologue code of the language
 *  - content: Content of all the source documents stored
 *  - Access ID & Key: Returned as a response after authentication
*/

import React, { Component } from 'react';
import './App.css';
import Header from './Header';
import Footer from './Footer';
import { FormControl } from 'react-bootstrap';
import Language from './SourceLanguages';
import TargetLanguages from './TargetLanguages';
import $ from 'jquery';

class UploadTokens extends Component {
  constructor(props) {
    super(props);

    this.state = {
      language:'tam',
      version: '',
      revision: '',
      targetlang: 'tam',
      tokenwords: {},
      uploaded:'Uploading',
      message: ''
    }

      // Upload file specific callback handlers
      this.uploadTokens = this.uploadTokens.bind(this);
      this.onSelect = this.onSelect.bind(this);
      this.file_xls = this.file_xls.bind(this);
  }
  
  onSelect(e) {
    this.setState({
      [e.target.name]: e.target.value });
  }

  file_xls(e, files){
    var _this = this;
    var file = files[0];
    var allRows = [];
    if (file) {
      var reader = new FileReader();
      reader.readAsText(file, "UTF-8");
      reader.onload = function (evt) {
          // console.log(evt.target.result.split("\n"));
         allRows = evt.target.result.split(/\r?\n|\r/);

      }
      reader.onerror = function (evt) {
          console.log("error reading file");
          allRows = []
      }
    }
    return allRows;
  }

  uploadTokens(e){
    var _this = this;
    let file = document.getElementById('file-input').files[0];
    var allRows = [];
    if (file) {
      var reader = new FileReader();
      reader.readAsText(file, "UTF-8");
      reader.onload = function (evt) {
          // console.log(evt.target.result.split("\n"));
         allRows = evt.target.result.split(/\r?\n|\r/);
        for(var singleRow = 0; singleRow < allRows.length; singleRow++) {
          let token = allRows[singleRow].split(",");
          _this.state.tokenwords[token[0]] = token[1];
        }
        
      }
      reader.onerror = function (evt) {
          console.log("error reading file");
          allRows = []
      }
    }
    e.preventDefault();

    console.log(this.state.language);
    console.log(this.state.version);
    console.log(this.state.revision);
    console.log(this.state.targetlang);
    console.log(this.state.tokenwords);

    var data = { <commit_msg>Added file for get language<commit_after>/**
 * @module src/UploadTokens
 *
 * Component that display SourceDocument
 * Accepts the following properties:
 *  - language: Ethnologue code of the language
 *  - content: Content of all the source documents stored
 *  - Access ID & Key: Returned as a response after authentication
*/

import React, { Component } from 'react';
import './App.css';
import Header from './Header';
import Footer from './Footer';
import { FormControl } from 'react-bootstrap';
import Language from './SourceLanguages';
import TargetLanguages from './TargetLanguages';
import $ from 'jquery';

class UploadTokens extends Component {
  constructor(props) {
    super(props);

    this.state = {
      language:'tam',
      version: '',
      revision: '',
      targetlang: 'tam',
      tokenwords: {},
      uploaded:'Uploading',
      message: ''
    }

      // Upload file specific callback handlers
      this.uploadTokens = this.uploadTokens.bind(this);
      this.onSelect = this.onSelect.bind(this);
      this.file_xls = this.file_xls.bind(this);
  }
  
  onSelect(e) {
    this.setState({
      [e.target.name]: e.target.value });
  }

  file_xls(e, files){
    var _this = this;
    var file = files[0];
    var allRows = [];
    if (file) {
      var reader = new FileReader();
      reader.readAsText(file, "UTF-8");
      reader.onload = function (evt) {
          // console.log(evt.target.result.split("\n"));
         allRows = evt.target.result.split(/\r?\n|\r/);

      }
      reader.onerror = function (evt) {
          console.log("error reading file");
          allRows = []
      }
    }
    return allRows;
  }

  uploadTokens(e){
    var _this = this;
    let file = document.getElementById('file-input').files[0];
    var allRows = [];
    if (file) {
      var reader = new FileReader();
      reader.readAsText(file, "UTF-8");
      reader.onload = function (evt) {
          // console.log(evt.target.result.split("\n"));
         allRows = evt.target.result.split(/\r?\n|\r/);
        for(var singleRow = 0; singleRow < allRows.length; singleRow++) {
          let token = allRows[singleRow].split(",");
          _this.state.tokenwords[token[0]] = token[1];
        }
        
      }
      reader.onerror = function (evt) {
          console.log("error reading file");
          allRows = []
      }
    }
    e.preventDefault();

    var data = { 
{'fraction_non_alphanumeric': 0.14589178356713428, 'fraction_numerical': 0.0028056112224448897, 'mean_word_length': 2.4468232044198897, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2535056', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1544, 'n_words': 351}