<commit_before><?php

/**
 * Copyright 2013 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

require_once 'vendor/autoload.php';

use Guzzle\Http\Client;
use fkooman\Config\Config;

try {
    $configFile = __DIR__ . DIRECTORY_SEPARATOR . "config" . DIRECTORY_SEPARATOR . "config.ini";
    $config = Config::fromIniFile($configFile);

    // data directory
    $exportDir      = $config->s('output')->l('exportDir', true); // REQ
    $metadataDir    = $config->s('output')->l('metadataDir', true); // REQ

    // remove all files from metadataDir
    foreach (glob($metadataDir . "/*.xml") as $f) {
        unlink($f);
    }

    $inputFile = $exportDir . DIRECTORY_SEPARATOR . "export.json";
    $exportData = @file_get_contents($inputFile);
    if (false === $exportData) {
        throw new Exception(sprintf("unable to read JSON file '%s' from disk", $inputFile));
    }
    $entities = json_decode($exportData, true);

    foreach ($entities as $type => $entity) {
        $metadataUrl = $entity['entityData']['metadataurl'];

        echo "'" . $metadataUrl . "'" . PHP_EOL;

        // create a hashed file name
        $fileName = $metadataDir . DIRECTORY_SEPARATOR . md5($metadataUrl) . ".xml";

        if (!file_exists($fileName)) {
            $md = fetchMetadata($metadataUrl);
            if (false === @file_put_contents($fileName, $md)) {
                throw new Exception(sprintf("unable to write metadata to file '%s'", $fileName));
            }
        }
    }

} catch (Exception $e) {
    echo sprintf("ERROR: %s", $e->getMessage());
    die(PHP_EOL);
}

function fetchMetadata($metadataUrl)
{
    try {
        $client = new Client(
            $metadataUrl,
            array(
                'curl.options' => array(
                    CURLOPT_CONNECTTIMEOUT => 10,
                    CURLOPT_TIMEOUT => 15
                ),
            )
        );
        $request = $client->get();
        $response = $request->send();
<commit_msg>Set CURLOPT_FOLLOWLOCATION to correctly handle redirects in metadata.php<commit_after><?php

/**
 * Copyright 2013 <NAME> <<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

require_once 'vendor/autoload.php';

use Guzzle\Http\Client;
use fkooman\Config\Config;

try {
    $configFile = __DIR__ . DIRECTORY_SEPARATOR . "config" . DIRECTORY_SEPARATOR . "config.ini";
    $config = Config::fromIniFile($configFile);

    // data directory
    $exportDir      = $config->s('output')->l('exportDir', true); // REQ
    $metadataDir    = $config->s('output')->l('metadataDir', true); // REQ

    // remove all files from metadataDir
    foreach (glob($metadataDir . "/*.xml") as $f) {
        unlink($f);
    }

    $inputFile = $exportDir . DIRECTORY_SEPARATOR . "export.json";
    $exportData = @file_get_contents($inputFile);
    if (false === $exportData) {
        throw new Exception(sprintf("unable to read JSON file '%s' from disk", $inputFile));
    }
    $entities = json_decode($exportData, true);

    foreach ($entities as $type => $entity) {
        $metadataUrl = $entity['entityData']['metadataurl'];

        echo "'" . $metadataUrl . "'" . PHP_EOL;

        // create a hashed file name
        $fileName = $metadataDir . DIRECTORY_SEPARATOR . md5($metadataUrl) . ".xml";

        if (!file_exists($fileName)) {
            $md = fetchMetadata($metadataUrl);
            if (false === @file_put_contents($fileName, $md)) {
                throw new Exception(sprintf("unable to write metadata to file '%s'", $fileName));
            }
        }
    }

} catch (Exception $e) {
    echo sprintf("ERROR: %s", $e->getMessage());
    die(PHP_EOL);
}

function fetchMetadata($metadataUrl)
{
    try {
        $client = new Client(
            $metadataUrl,
            array(
                'curl.options' => array(
                    CURLOPT_CONNECTTIMEOUT => 10,
                    CURLOPT_TIMEOUT => 15,
                    CURLOPT_FOLLOWLOCATION => TRUE
                ),
            )
        );
        $request = $client->get();
        $response = $request->send();

{'fraction_non_alphanumeric': 0.13390928725701945, 'fraction_numerical': 0.005105046141763205, 'mean_word_length': 2.5672268907563027, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6905750', 'n_tokens_mistral': 1500, 'n_tokens_neox': 1465, 'n_words': 421}