<commit_before>/*

   Copyright 2015-2016 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package org.blitzortung.android.data.provider.blitzortung

import android.content.SharedPreferences
import android.util.Log
import org.blitzortung.android.app.Main
import org.blitzortung.android.app.view.PreferenceKey
import org.blitzortung.android.app.view.get
import org.blitzortung.android.data.Parameters
import org.blitzortung.android.data.beans.Station
import org.blitzortung.android.data.beans.Strike
import org.blitzortung.android.data.provider.DataProvider
import org.blitzortung.android.data.provider.DataProviderType
import org.blitzortung.android.data.provider.result.ResultEvent
import java.io.BufferedReader
import java.net.*
import java.util.*
import java.util.zip.GZIPInputStream

class BlitzortungHttpDataProvider @JvmOverloads constructor(
        preferences: SharedPreferences,
        private val urlFormatter: UrlFormatter = UrlFormatter(),
        mapBuilderFactory: MapBuilderFactory = MapBuilderFactory()
) : DataProvider(preferences, PreferenceKey.USERNAME, PreferenceKey.PASSWORD) {

    private val strikeMapBuilder: MapBuilder<Strike>
    private val stationMapBuilder: MapBuilder<Station>
    private var latestTime: Long = 0
    private var strikes: List<Strike> = emptyList()
    private var parameters: Parameters? = null

    private lateinit var username: String
    private lateinit var password: String

    init {
        strikeMapBuilder = mapBuilderFactory.createAbstractStrikeMapBuilder()
        stationMapBuilder = mapBuilderFactory.createStationMapBuilder()
    }

    private fun readFromUrl(type: Type, region: Int, intervalTime: Calendar? = null): BufferedReader? {

        val useGzipCompression = type == Type.STATIONS

        val reader: BufferedReader

        val urlString = urlFormatter.getUrlFor(type, region, intervalTime, useGzipCompression)

        Log.v(Main.LOG_TAG, "BlitzortungHttpDataProvider.readFromUrl() $urlString")

        try {
            val url: URL
            url = URL(urlString)
            val connection = url.openConnection()
            connection.connectTimeout = 40000
            connection.readTimeout = 40000
            connection.allowUserInteraction = false
            var inputStream = connection.inputStream
            if (useGzipCompression) {
                inputStream = GZIPInputStream(inputStream)
            }

            reader = inputStream.bufferedReader()
        } catch (e: Exception) {
            Log.w(Main.LOG_TAG, "BlitzortungHttpDataProvider.readFromUrl() URL $urlString failed")
            throw RuntimeException(e)
        }

        return reader
    }

    /**
     * Used to retrieve Data from Blitzortung
     * @param readerSeq A sequence of nullable BufferedReader, which the data is read from
     * @param parse A Lambda which receives a sequence of lines from a buffered reader and transforms them into a sequence of T
     * @return Returns a list of parsed T's
     */
    private fun <T : Any> retrieveData(logMessage: String, readerSeq: Sequence<BufferedReader?>, parse: (String) -> T?): List<T> {
        var size = 0

        val strikeSequence: Sequence<T> = readerSeq.filterNotNull().flatMap { reader ->
            reader.lineSequence().mapNotNull { line ->
                size += line.length

                parse(line)
            }
        }

        val strikeList = try {
            strikeSequence.toList()
        } catch (e: SocketException) {
            Log.w(Main.LOG_TAG, e);<commit_msg>handle non existent Blitzortung data files<commit_after>/*

   Copyright 2015-2016 <NAME>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package org.blitzortung.android.data.provider.blitzortung

import android.content.SharedPreferences
import android.util.Log
import org.blitzortung.android.app.Main
import org.blitzortung.android.app.view.PreferenceKey
import org.blitzortung.android.app.view.get
import org.blitzortung.android.data.Parameters
import org.blitzortung.android.data.beans.Station
import org.blitzortung.android.data.beans.Strike
import org.blitzortung.android.data.provider.DataProvider
import org.blitzortung.android.data.provider.DataProviderType
import org.blitzortung.android.data.provider.result.ResultEvent
import java.io.BufferedReader
import java.io.ByteArrayInputStream
import java.io.FileNotFoundException
import java.io.InputStreamReader
import java.net.*
import java.util.*
import java.util.zip.GZIPInputStream

class BlitzortungHttpDataProvider @JvmOverloads constructor(
        preferences: SharedPreferences,
        private val urlFormatter: UrlFormatter = UrlFormatter(),
        mapBuilderFactory: MapBuilderFactory = MapBuilderFactory()
) : DataProvider(preferences, PreferenceKey.USERNAME, PreferenceKey.PASSWORD) {

    private val strikeMapBuilder: MapBuilder<Strike>
    private val stationMapBuilder: MapBuilder<Station>
    private var latestTime: Long = 0
    private var strikes: List<Strike> = emptyList()
    private var parameters: Parameters? = null

    private lateinit var username: String
    private lateinit var password: String

    init {
        strikeMapBuilder = mapBuilderFactory.createAbstractStrikeMapBuilder()
        stationMapBuilder = mapBuilderFactory.createStationMapBuilder()
    }

    private fun readFromUrl(type: Type, region: Int, intervalTime: Calendar? = null): BufferedReader? {

        val useGzipCompression = type == Type.STATIONS

        val reader: BufferedReader

        val urlString = urlFormatter.getUrlFor(type, region, intervalTime, useGzipCompression)



        try {
            val url: URL
            url = URL(urlString)
            val connection = url.openConnection()
            connection.connectTimeout = 40000
            connection.readTimeout = 40000
            connection.allowUserInteraction = false
            var inputStream = connection.inputStream
            if (useGzipCompression) {
                inputStream = GZIPInputStream(inputStream)
            }

            reader = inputStream.bufferedReader()
        } catch (e: Exception) {
            when (e) {
                is FileNotFoundException -> {
                    Log.w(Main.LOG_TAG, "BlitzortungHttpDataProvider.readFromUrl() $urlString not found")
                    return BufferedReader(InputStreamReader(ByteArrayInputStream("".toByteArray())))
                }
                else -> {
                    Log.w(Main.LOG_TAG, "BlitzortungHttpDataProvider.readFromUrl() $urlString failed")
                    throw RuntimeException(e)
                }
            }
        }

        Log.v(Main.LOG_TAG, "BlitzortungHttpDataProvider.readFromUrl() $urlString")

        return reader
    }

    /**
     * Used to retrieve Data from Blitzortung
     * @param readerSeq A sequence of nullable BufferedReader, which the data is read from
     * @param parse A Lambda which receives a sequence of lines from a buffered reader and transforms them into a sequence of T
     * @return Returns a list of parsed T's
     */
    private fun <T : Any> retrieveData(logMessage: String, readerSeq: Sequence<BufferedReader?>, parse: (String) -> T?): List<T> {
        var size = 0

        val strikeSequence: Sequence<T> = readerSeq.filterNotNull().flatMap { reader ->
            reader.lineSequence().mapNotNull { line ->
                size += line.length

                parse(line)
            }
        }

        val strikeList = try {
            strikeSequence.toList()
        } catch (e: SocketException) {
            Log.w(Main.LOG_TAG, e);
{'fraction_non_alphanumeric': 0.07682458386683738, 'fraction_numerical': 0.005587242463042719, 'mean_word_length': 2.950344827586207, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '424857', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2228, 'n_words': 647}