<commit_before># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
#
# Download and build the data if it does not exist.

import parlai.core.build_data as build_data
import os


def build(opt):
    dpath = os.path.join(opt['datapath'], 'wmt')
    version = 'None'

    if not build_data.built(dpath, version_string=version):
        print('[building data: ' + dpath + ']')
        if build_data.built(dpath):
            # An older version exists, so remove these outdated files.
            build_data.remove_dir(dpath)
        build_data.make_dir(dpath)

        # Download the data.
        fnames = [('train.en','train.de', 'en_de_train.txt'),
        ('newstest2014.en','newstest2014.de', 'en_de_test.txt')]
        for (en_fname, de_fname, w_fname) in fnames:
            url_base = 'https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/'
            en_url = url_base + en_fname
            de_url = url_base + de_fname
            build_data.download(en_url, dpath, en_fname)
            build_data.download(de_url, dpath, de_fname)
            with open(os.path.join(dpath, en_fname), 'r') as f:
                # We replace '##AT##-##AT##' as a workaround in order to use the
                # nltk tokenizer specified by DictionaryAgent
                en = [l[:-1].replace("##AT##-##AT##", "__AT__") for l in f]

            with open(os.path.join(dpath, de_fname), 'r') as f:
                de = [l[:-1].replace("##AT##-##AT##", "__AT__") for l in f]

            with open(os.path.join(dpath, w_fname), 'w') as f:
              for de_sent,en_sent in zip(de,en):
                f.write("1 "+en_sent+"\t"+de_sent+"\n")


        # Mark the data as built.
        build_data.mark_done(dpath, version_string=version)<commit_msg>add wmt:en_de val set (#359)<commit_after># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
#
# Download and build the data if it does not exist.

import parlai.core.build_data as build_data
import os
import numpy


def readFiles(dpath, rfnames):
    en_fname, de_fname = rfnames
    url_base = 'https://nlp.stanford.edu/projects/nmt/data/wmt14.en-de/'
    en_url = url_base + en_fname
    de_url = url_base + de_fname
    build_data.download(en_url, dpath, en_fname)
    build_data.download(de_url, dpath, de_fname)
    with open(os.path.join(dpath, en_fname), 'r') as f:
        # We replace '##AT##-##AT##' as a workaround in order to use the
        # nltk tokenizer specified by DictionaryAgent
        en = [l[:-1].replace("##AT##-##AT##", "__AT__") for l in f]

    with open(os.path.join(dpath, de_fname), 'r') as f:
        de = [l[:-1].replace("##AT##-##AT##", "__AT__") for l in f]

    return list(zip(de, en))


def build(opt):
    dpath = os.path.join(opt['datapath'], 'wmt')
    version = 'None'

    if not build_data.built(dpath, version_string=version):
        print('[building data: ' + dpath + ']')
        if build_data.built(dpath):
            # An older version exists, so remove these outdated files.
            build_data.remove_dir(dpath)
        build_data.make_dir(dpath)

        # Download the data.

        train_r_fnames = ('train.en', 'train.de')
        train_w_fname = 'en_de_train.txt'
        valid_w_fname = 'en_de_valid.txt'
        test_r_fnames = ('newstest2014.en', 'newstest2014.de')
        test_w_fname = 'en_de_test.txt'

        train_zip = readFiles(dpath, train_r_fnames)
        numpy.random.shuffle(train_zip)
        with open(os.path.join(dpath, valid_w_fname), 'w') as f:
            for de_sent, en_sent in train_zip[:30000]:
                f.write("1 "+en_sent+"\t"+de_sent+"\n")
        with open(os.path.join(dpath, train_w_fname), 'w') as f:
            for de_sent, en_sent in train_zip[30000:]:
                f.write("1 "+en_sent+"\t"+de_sent+"\n")

        test_zip = readFiles(dpath, test_r_fnames)
        with open(os.path.join(dpath, test_w_fname), 'w') as f:
            for de_sent, en_sent in test_zip:
                f.write("1 "+en_sent+"\t"+de_sent+"\n")

        # Mark the data as built.
        build_data.mark_done(dpath, version_string=version)
{'fraction_non_alphanumeric': 0.13147497805092187, 'fraction_numerical': 0.010755048287971905, 'mean_word_length': 2.7018683996750608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7117664', 'n_tokens_mistral': 1622, 'n_tokens_neox': 1560, 'n_words': 409}