<commit_before>/* Copyright 2002-2009 CS Communication & Systèmes
 * Licensed to CS Communication & Systèmes (CS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * CS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.orekit.data;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.text.ParseException;
import java.util.regex.Pattern;

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;

import org.orekit.errors.OrekitException;

public class NetworkCrawlerTest extends TestCase {

    public void testNoElement() {
        File existing   = new File(url("regular-data").getPath());
        File inexistent = new File(existing.getParent(), "inexistant-directory");
        try {
            new NetworkCrawler(inexistent.toURI().toURL()).feed(new CountingLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException e) {
            // expected behavior
        } catch (Exception e) {
            e.printStackTrace();
            fail("wrong exception caught");
        }
    }

    // WARNING!
    // the following test is commented out by default, as it does connect to the web
    // if you want to enable it, you will have uncomment it and to either set the proxy
    // settings according to your local network or remove the proxy authentication
    // settings if you have a transparent connection to internet
//    public void testRemote() throws java.net.MalformedURLException, OrekitException {
//
//        System.setProperty("http.proxyHost",     "proxy.your.domain.com");
//        System.setProperty("http.proxyPort",     "8080");
//        System.setProperty("http.nonProxyHosts", "localhost|*.your.domain.com");
//        java.net.Authenticator.setDefault(new AuthenticatorDialog());
//
//        CountingLoader loader = new CountingLoader(".*\\.history");
//        DataWebCrawler crawler =
//            new DataWebCrawler(new URL("http://hpiers.obspm.fr/eoppc/bul/bulc/UTC-TAI.history"));
//        crawler.setTimeout(1000);
//        crawler.feed(loader);
//        assertEquals(1, loader.getCount());
//
//    }

    public void testLocal() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*");
        new NetworkCrawler(url("regular-data/UTC-TAI.history"),
                           url("regular-data/de405-ephemerides/unxp0000.405"),
                           url("regular-data/de405-ephemerides/unxp0001.405"),
                           url("regular-data/de406-ephemerides/unxp0000.406"),
                           url("regular-data/Earth-orientation-parameters/monthly/bulletinb_IAU2000-216.txt"),
                           url("no-data")).feed(crawler);
        assertEquals(6, crawler.getCount());
    }

    public void testCompressed() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*/eopc04.*");
        new NetworkCrawler(url("compressed-data/UTC-TAI.history.gz"),
                           url("compressed-data/eopc04_IAU2000.00.gz"),
                           url("compressed-data/eopc04_IAU2000.02.gz")).feed(crawler);
        assertEquals(2, crawler.getCount());
    }

    public void testMultiZip() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*\\.txt$");
        new NetworkCrawler(url("zipped-data/multizip.zip")).feed(crawler);
        assertEquals(6, crawler.getCount());
    }

    public void testIOException() throws OrekitException {
        try {
            new NetworkCrawler(url("regular-data/UTC-TAI.history")).feed(new IOExceptionLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException oe) {
            // expected behavior
            assertNotNull(oe.getCause());
            assertEquals(IOException.class, oe.getCause().getClass());
            assertEquals("dummy error", oe.getMessage());
        } catch (Exception e) {
            fail("wrong exception caught");
        }
    }

    public void testParseException() throws OrekitException {
        try {
            new NetworkCrawler(url("regular-data/UTC-TAI.history")).feed(new ParseExceptionLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException oe) {
            // expected behavior
            assertNotNull(oe.getCause());
            assertEquals(ParseException.class, oe.getCause().getClass());
            assertEquals("dummy error", oe.getMessage());
        } catch (Exception e) {
            e.printStackTrace(System.err);
            fail("wrong exception caught");
        }
    }

    private static class CountingLoader implements DataLoader {
        private Pattern namePattern;
        private int count;
        public CountingLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
            count = 0;
        }
        public void loadData(InputStream input, String name) {
            ++count;
        }
        public int getCount() {
            return count;
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private static class IOExceptionLoader implements DataLoader {
        private Pattern namePattern;
        public IOExceptionLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
        }
        public void loadData(InputStream input, String name) throws IOException {
            if (name.endsWith("UTC-TAI.history")) {
                throw new IOException("dummy error");
            }
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private static class ParseExceptionLoader implements DataLoader {
        private Pattern namePattern;
        public ParseExceptionLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
        }
        public void loadData(InputStream input, String name) throws ParseException {
            if (name.endsWith("UTC-TAI.history")) {
                throw new ParseException("dummy error", 0);
            }
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private URL url(String resource) {
        return DirectoryCrawlerTest.class.getClassLoader().getResource(resource);
    }

    public static Test suite() {
        return new TestSuite(NetworkCrawlerTest.class);
    }

}
<commit_msg>changed to Junit 4 updated remote test with current class names<commit_after>/* Copyright 2002-2009 CS Communication & Systèmes
 * Licensed to CS Communication & Systèmes (CS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * CS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.orekit.data;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.text.ParseException;
import java.util.regex.Pattern;

import org.junit.Test;
import org.orekit.errors.OrekitException;

public class NetworkCrawlerTest {

    @Test
    public void noElement() {
        File existing   = new File(url("regular-data").getPath());
        File inexistent = new File(existing.getParent(), "inexistant-directory");
        try {
            new NetworkCrawler(inexistent.toURI().toURL()).feed(new CountingLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException e) {
            // expected behavior
        } catch (Exception e) {
            e.printStackTrace();
            fail("wrong exception caught");
        }
    }

    // WARNING!
    // the following test is commented out by default, as it does connect to the web
    // if you want to enable it, you will have uncomment it and to either set the proxy
    // settings according to your local network or remove the proxy authentication
    // settings if you have a transparent connection to internet
//    @Test
//    public void remote() throws java.net.MalformedURLException, OrekitException {
//
//        System.setProperty("http.proxyHost",     "proxy.your.domain.com");
//        System.setProperty("http.proxyPort",     "8080");
//        System.setProperty("http.nonProxyHosts", "localhost|*.your.domain.com");
//        java.net.Authenticator.setDefault(new AuthenticatorDialog());
//
//        CountingLoader loader = new CountingLoader(".*\\.history");
//        NetworkCrawler crawler =
//            new NetworkCrawler(new URL("http://hpiers.obspm.fr/eoppc/bul/bulc/UTC-TAI.history"));
//        crawler.setTimeout(1000);
//        crawler.feed(loader);
//        assertEquals(1, loader.getCount());
//
//    }

    @Test
    public void local() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*");
        new NetworkCrawler(url("regular-data/UTC-TAI.history"),
                           url("regular-data/de405-ephemerides/unxp0000.405"),
                           url("regular-data/de405-ephemerides/unxp0001.405"),
                           url("regular-data/de406-ephemerides/unxp0000.406"),
                           url("regular-data/Earth-orientation-parameters/monthly/bulletinb_IAU2000-216.txt"),
                           url("no-data")).feed(crawler);
        assertEquals(6, crawler.getCount());
    }

    @Test
    public void compressed() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*/eopc04.*");
        new NetworkCrawler(url("compressed-data/UTC-TAI.history.gz"),
                           url("compressed-data/eopc04_IAU2000.00.gz"),
                           url("compressed-data/eopc04_IAU2000.02.gz")).feed(crawler);
        assertEquals(2, crawler.getCount());
    }

    @Test
    public void multiZip() throws OrekitException {
        CountingLoader crawler = new CountingLoader(".*\\.txt$");
        new NetworkCrawler(url("zipped-data/multizip.zip")).feed(crawler);
        assertEquals(6, crawler.getCount());
    }

    @Test
    public void ioException() throws OrekitException {
        try {
            new NetworkCrawler(url("regular-data/UTC-TAI.history")).feed(new IOExceptionLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException oe) {
            // expected behavior
            assertNotNull(oe.getCause());
            assertEquals(IOException.class, oe.getCause().getClass());
            assertEquals("dummy error", oe.getMessage());
        } catch (Exception e) {
            fail("wrong exception caught");
        }
    }

    @Test
    public void parseException() throws OrekitException {
        try {
            new NetworkCrawler(url("regular-data/UTC-TAI.history")).feed(new ParseExceptionLoader(".*"));
            fail("an exception should have been thrown");
        } catch (OrekitException oe) {
            // expected behavior
            assertNotNull(oe.getCause());
            assertEquals(ParseException.class, oe.getCause().getClass());
            assertEquals("dummy error", oe.getMessage());
        } catch (Exception e) {
            e.printStackTrace(System.err);
            fail("wrong exception caught");
        }
    }

    private static class CountingLoader implements DataLoader {
        private Pattern namePattern;
        private int count;
        public CountingLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
            count = 0;
        }
        public void loadData(InputStream input, String name) {
            ++count;
        }
        public int getCount() {
            return count;
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private static class IOExceptionLoader implements DataLoader {
        private Pattern namePattern;
        public IOExceptionLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
        }
        public void loadData(InputStream input, String name) throws IOException {
            if (name.endsWith("UTC-TAI.history")) {
                throw new IOException("dummy error");
            }
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private static class ParseExceptionLoader implements DataLoader {
        private Pattern namePattern;
        public ParseExceptionLoader(String pattern) {
            namePattern = Pattern.compile(pattern);
        }
        public void loadData(InputStream input, String name) throws ParseException {
            if (name.endsWith("UTC-TAI.history")) {
                throw new ParseException("dummy error", 0);
            }
        }
        public boolean fileIsSupported(String fileName) {
            return namePattern.matcher(fileName).matches();
        }
    }

    private URL url(String resource) {
        return DirectoryCrawlerTest.class.getClassLoader().getResource(resource);
    }

}

{'fraction_non_alphanumeric': 0.106504400440044, 'fraction_numerical': 0.011207370737073707, 'mean_word_length': 2.4721890666030077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7553998', 'n_tokens_mistral': 3868, 'n_tokens_neox': 3738, 'n_words': 987}