<commit_before><?php
/**
 * Tatoeba Project, free collaborative creation of multilingual corpuses project
 * Copyright (C) 2009  <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * PHP version 5
 *
 * @category PHP
 * @package  Tatoeba
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

$this->pageTitle = 'Tatoeba - ' . __('Download sentences', true);
?>

<div id="annexe_content">
    <?php
    $attentionPlease->tatoebaNeedsYou();
    ?>
    
    <div class="module">
        <h2>Warning</h2>
        <p>
            The data you will find here will NOT be useful unless you are coding a
            language tool or doing some work on data processing.
        </p>
        <p>
            If you want data that you can use as a humble language learner, you can
            check out the 
            <?php 
            echo $html->link(
                'lists section', array("controller"=>"sentences_lists")
            );
            ?> 
            where you can build your own lists of sentences or view others' lists 
            and print them.
        </p>
    </div>
    
    <div class="module">
        <h2>Creative commons</h2>
        <p>These files are released under CC-BY.</p>
        <a rel="license" href="http://creativecommons.org/licenses/by/2.0/fr/">
            <img alt="Creative Commons License" style="border-width:0"
                src="http://i.creativecommons.org/l/by/2.0/fr/88x31.png" />
        </a>
    
        <p>
            For those who wonder why we're not leaving the data in the public
            domain, some explanations
            <a href="http://blog.tatoeba.org/2009/12/tatoeba-update-dec-12th-2009.html">
            here</a>.
        </p>
    </div>
    
    <div class="module">
        <h2>Questions?</h2>
        <p>
            If you have questions or requests, feel free to 
            <?php
            echo $html->link(
                "contact us", array("controller"=>"pages", "action"=>"contact")
            );
            ?>. In general we answer quickly.
        </p>
    </div>
</div>

<div id="main_content">
    
    <div class="module">
        <h2>Downloads</h2>
        
        <!-- Sentences -->
        <h3>Sentences</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                1. <a href="http://tatoeba.org/files/downloads/sentences.csv">
                http://tatoeba.org/files/downloads/sentences.csv
                </a>
            </dd>
            <dd>
                2. <a href="http://tatoeba.org/files/downloads/sentences_detailed.csv">
                http://tatoeba.org/files/downloads/sentences_detailed.csv
                </a>
            </dd>
            <dt>Fields and structure</dt>
            <dd>
                1. <span class="param">id</span>
                <span class="symbol">[tab]</span>
                <span class="param">lang</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span>
            </dd>
            <dd>
                2. <span class="param">id</span>
                <span class="symbol">[tab]</span>
                <span class="param">lang</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span>
                <span class="symbol">[tab]</span>
                <span class="param">username</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_added</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_last_modified</span>                
            </dd>


            <dt>Description</dt>
            <dd>
                Contains all the sentences. Each sentence is associated to a
                unique id and a language code 
                (<a href="http://en.wikipedia.org/wiki/List_of_ISO_639-3_codes">ISO 639-3</a>).
                <br/>
                We provide two files. The first file (sentences.csv) only contains the minimum.
                The second file (sentences_detailed.csv) contains more information, for those 
                who would like to filter the sentences based, for instance, on the contributor 
                who owns the sentence or on the date when it was added.
            </dd>
        </dl>
        
        <!-- Links -->
        <h3>Links</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/links.csv">
                http://tatoeba.org/files/downloads/links.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">translation_id</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the links between the sentences. 
                <span class="param">1</span>
                <span class="symbol">[tab]</span>
                <span class="param">77</span> 
                means that sentence nº77 is the translation of sentence nº1. 
                The reciprocal link is also present. 
                In other words, you will also have a line that says
                <span class="param">77</span>
                <span class="symbol">[tab]</span>
                <span class="param">1</span>.
            </dd>
        </dl>
        
        
        <!-- Tags -->
        <h3>Tags</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/tags.csv">
                http://tatoeba.org/files/downloads/tags.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">tag_name</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the list of tags associated to each sentence. 
                <span class="param">381279</span>
                <span class="symbol">[tab]</span>
                <span class="param">proverb</span> 
                means that sentence nº381279 has been tagged with "proverb".
            </dd>
        </dl>
        
        
        <!-- Lists -->
        <h3>Lists</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/lists.csv">
                http://tatoeba.org/files/downloads/lists.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">list_name</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the list of sentence lists associated to each sentence. 
                <span class="param">381279</span>
                <span class="symbol">[tab]</span>
                <span class="param">My own list</span> 
                means that sentence nº381279 is part of "My own list".
            </dd>
        </dl>
            
            
        <!-- Indices -->
        <h3>Japanese indices</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/jpn_indices.csv">
                http://tatoeba.org/files/downloads/jpn_indices.csv
                </a>
            </dd>
            

            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">meaning_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span><commit_msg>Updated downloads page to take into account new lists exports.<commit_after><?php
/**
 * Tatoeba Project, free collaborative creation of multilingual corpuses project
 * Copyright (C) 2009  <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * PHP version 5
 *
 * @category PHP
 * @package  Tatoeba
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

$this->pageTitle = 'Tatoeba - ' . __('Download sentences', true);
?>

<div id="annexe_content">
    <?php
    $attentionPlease->tatoebaNeedsYou();
    ?>
    
    <div class="module">
        <h2>Warning</h2>
        <p>
            The data you will find here will NOT be useful unless you are coding a
            language tool or doing some work on data processing.
        </p>
        <p>
            If you want data that you can use as a humble language learner, you can
            check out the 
            <?php 
            echo $html->link(
                'lists section', array("controller"=>"sentences_lists")
            );
            ?> 
            where you can build your own lists of sentences or view others' lists 
            and print them.
        </p>
    </div>
    
    <div class="module">
        <h2>Creative commons</h2>
        <p>These files are released under CC-BY.</p>
        <a rel="license" href="http://creativecommons.org/licenses/by/2.0/fr/">
            <img alt="Creative Commons License" style="border-width:0"
                src="http://i.creativecommons.org/l/by/2.0/fr/88x31.png" />
        </a>
    
        <p>
            For those who wonder why we're not leaving the data in the public
            domain, some explanations
            <a href="http://blog.tatoeba.org/2009/12/tatoeba-update-dec-12th-2009.html">
            here</a>.
        </p>
    </div>
    
    <div class="module">
        <h2>Questions?</h2>
        <p>
            If you have questions or requests, feel free to 
            <?php
            echo $html->link(
                "contact us", array("controller"=>"pages", "action"=>"contact")
            );
            ?>. In general we answer quickly.
        </p>
    </div>
</div>

<div id="main_content">
    
    <div class="module">
        <h2>Downloads</h2>
        
        <!-- Sentences -->
        <h3>Sentences</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                1. <a href="http://tatoeba.org/files/downloads/sentences.csv">
                http://tatoeba.org/files/downloads/sentences.csv
                </a>
            </dd>
            <dd>
                2. <a href="http://tatoeba.org/files/downloads/sentences_detailed.csv">
                http://tatoeba.org/files/downloads/sentences_detailed.csv
                </a>
            </dd>
            <dt>Fields and structure</dt>
            <dd>
                1. <span class="param">id</span>
                <span class="symbol">[tab]</span>
                <span class="param">lang</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span>
            </dd>
            <dd>
                2. <span class="param">id</span>
                <span class="symbol">[tab]</span>
                <span class="param">lang</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span>
                <span class="symbol">[tab]</span>
                <span class="param">username</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_added</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_last_modified</span>                
            </dd>


            <dt>Description</dt>
            <dd>
                Contains all the sentences. Each sentence is associated to a
                unique id and a language code 
                (<a href="http://en.wikipedia.org/wiki/List_of_ISO_639-3_codes">ISO 639-3</a>).
                <br/>
                We provide two files. The first file (sentences.csv) only contains the minimum.
                The second file (sentences_detailed.csv) contains more information, for those 
                who would like to filter the sentences based, for instance, on the contributor 
                who owns the sentence or on the date when it was added.
            </dd>
        </dl>
        
        <!-- Links -->
        <h3>Links</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/links.csv">
                http://tatoeba.org/files/downloads/links.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">translation_id</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the links between the sentences. 
                <span class="param">1</span>
                <span class="symbol">[tab]</span>
                <span class="param">77</span> 
                means that sentence nº77 is the translation of sentence nº1. 
                The reciprocal link is also present. 
                In other words, you will also have a line that says
                <span class="param">77</span>
                <span class="symbol">[tab]</span>
                <span class="param">1</span>.
            </dd>
        </dl>
        
        
        <!-- Tags -->
        <h3>Tags</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/tags.csv">
                http://tatoeba.org/files/downloads/tags.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">tag_name</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the list of tags associated to each sentence. 
                <span class="param">381279</span>
                <span class="symbol">[tab]</span>
                <span class="param">proverb</span> 
                means that sentence nº381279 has been tagged with "proverb".
            </dd>
        </dl>
        
        
        <!-- Lists -->
        <h3>Lists</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/user_lists.csv">
                http://tatoeba.org/files/downloads/user_lists.csv
                </a>
            </dd>
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">id</span>
                <span class="symbol">[tab]</span>
                <span class="param">username</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_created</span>
                <span class="symbol">[tab]</span>
                <span class="param">date_modified</span>
                <span class="symbol">[tab]</span>
                <span class="param">list_name</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Contains the list of <a href="http://tatoeba.org/sentences_lists/index">
                public lists</a> created.
            </dd>
          </dl>
          
          <h3>Sentences in lists</h3>
          <dl>       
            <dt>Download</dt>   
            <dd>
                <a href="http://tatoeba.org/files/downloads/sentences_in_lists.csv">
                http://tatoeba.org/files/downloads/sentences_in_lists.csv
                </a>
            </dd>
            
            <dt>Fields and structure</dt>
            <dd>
                <span class="param">list_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">sentence_id</span>
            </dd>
            
            <dt>Description</dt>
            <dd>
                Indicates the sentences that are in each of the public lists. 
                <span class="param">13</span>
                <span class="symbol">[tab]</span>
                <span class="param">381279</span> 
                means that sentence nº381279 is part of the list of id 13.
            </dd>
        </dl>
            
            
        <!-- Indices -->
        <h3>Japanese indices</h3>
        <dl>
            <dt>Download</dt>
            <dd>
                <a href="http://tatoeba.org/files/downloads/jpn_indices.csv">
                http://tatoeba.org/files/downloads/jpn_indices.csv
                </a>
            </dd>
            

            <dt>Fields and structure</dt>
            <dd>
                <span class="param">sentence_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">meaning_id</span>
                <span class="symbol">[tab]</span>
                <span class="param">text</span>
{'fraction_non_alphanumeric': 0.12885848192641275, 'fraction_numerical': 0.010019932123040457, 'mean_word_length': 1.4228660924040721, 'pattern_counts': {'":': 0, '<': 493, '<?xml version=': 0, '>': 507, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7387376', 'n_tokens_mistral': 5420, 'n_tokens_neox': 5046, 'n_words': 1288}