<commit_before>/**
  * Copyright 2015 <NAME>
  *
  * Licensed under the Apache License, Version 2.0 (the “License”); you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  * an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  *
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
package cmwell.web.ld.util

import java.io.InputStream

import cmwell.domain._
import cmwell.fts._
import cmwell.util.string.Hash._
import cmwell.util.string._
import cmwell.web.ld.cmw.CMWellRDFHelper.{Create, Exists, PrefixState}
import cmwell.web.ld.cmw._
import cmwell.web.ld.exceptions._
import cmwell.ws.Settings
import cmwell.common.{Settings => CSettings}
import com.typesafe.scalalogging.LazyLogging
import logic.{CRUDServiceFS, InfotonValidator}
import org.apache.jena.datatypes.xsd.XSDDatatype
import org.apache.jena.datatypes.xsd.impl.RDFLangString
import org.apache.jena.graph.BlankNodeId
import org.apache.jena.query.DatasetFactory
import org.apache.jena.rdf.model.{Seq => _, _}
import org.apache.jena.riot.{Lang, RDFDataMgr}
import org.apache.xerces.util.XMLChar
import security.{AuthUtils, PermissionLevel, Token}
import wsutil._

import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.collection.immutable.{Map => IMap, Set => ISet}
import scala.collection.mutable.{ListBuffer, Map => MMap, Set => MSet}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success, Try}

/**
  * Created with IntelliJ IDEA.
  * User: gilad
  * Date: 7/10/13
  * Time: 12:16 PM
  * To change this template use File | Settings | File Templates.
  */
object LDFormatParser extends LazyLogging {

  private val http = "http://"
  private val https = "https://"
  private val someHttps = Some("https://")
  private val cmwell = "cmwell://"
  private val blank = cmwell + "blank_node/"
  private lazy val cwd = new java.io.File("").toURI
  private lazy val normalizedCWD = {
    val fileURI = cwd.toString
    if (fileURI.startsWith("file:///")) fileURI
    else {
      var charsSeen = 0
      "file:///" + fileURI.drop("file:".length).dropWhile { ch =>
        charsSeen < 3 && {
          charsSeen += 1
          ch == '/'
        }
      }
    }
  }
  val urlFKey = NnFieldKey("url")
  val prefixFKey = NnFieldKey("prefix")
  val aliasFKey = NnFieldKey("alias")
  val graphFKey = NnFieldKey("graph")

  implicit class LegalChar(ch: Char) {
    // format: off
    def isLegal: Boolean = {
      val i = ch.toInt
      (45 to 57).contains(i)  ||     // (-, ., /, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
      (65 to 90).contains(i)  ||     // (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z)
      (97 to 122).contains(i) ||     // (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z)
      i == 95 || i == 35 || i > 126  // 95 = '_', 35 = '#', 123 to 126 are: ({, |, }, ~)
    }
    // format: on
  }

  def createFieldValue(l: Literal, q: Option[String]): FieldValue = {
    val lang = {
      val lng = l.getLanguage
      if (lng.isEmpty) None
      else Some(lng)<commit_msg>Allow percent, comma, and colon in URIs (#1123)<commit_after>/**
  * Copyright 2015 <NAME>
  *
  * Licensed under the Apache License, Version 2.0 (the “License”); you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  * an “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  *
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
package cmwell.web.ld.util

import java.io.InputStream

import cmwell.domain._
import cmwell.fts._
import cmwell.util.string.Hash._
import cmwell.util.string._
import cmwell.web.ld.cmw.CMWellRDFHelper.{Create, Exists, PrefixState}
import cmwell.web.ld.cmw._
import cmwell.web.ld.exceptions._
import cmwell.ws.Settings
import cmwell.common.{Settings => CSettings}
import com.typesafe.scalalogging.LazyLogging
import logic.{CRUDServiceFS, InfotonValidator}
import org.apache.jena.datatypes.xsd.XSDDatatype
import org.apache.jena.datatypes.xsd.impl.RDFLangString
import org.apache.jena.graph.BlankNodeId
import org.apache.jena.query.DatasetFactory
import org.apache.jena.rdf.model.{Seq => _, _}
import org.apache.jena.riot.{Lang, RDFDataMgr}
import org.apache.xerces.util.XMLChar
import security.{AuthUtils, PermissionLevel, Token}
import wsutil._

import scala.annotation.tailrec
import scala.collection.JavaConversions._
import scala.collection.immutable.{Map => IMap, Set => ISet}
import scala.collection.mutable.{ListBuffer, Map => MMap, Set => MSet}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success, Try}

/**
  * Created with IntelliJ IDEA.
  * User: gilad
  * Date: 7/10/13
  * Time: 12:16 PM
  * To change this template use File | Settings | File Templates.
  */
object LDFormatParser extends LazyLogging {

  private val http = "http://"
  private val https = "https://"
  private val someHttps = Some("https://")
  private val cmwell = "cmwell://"
  private val blank = cmwell + "blank_node/"
  private lazy val cwd = new java.io.File("").toURI
  private lazy val normalizedCWD = {
    val fileURI = cwd.toString
    if (fileURI.startsWith("file:///")) fileURI
    else {
      var charsSeen = 0
      "file:///" + fileURI.drop("file:".length).dropWhile { ch =>
        charsSeen < 3 && {
          charsSeen += 1
          ch == '/'
        }
      }
    }
  }
  val urlFKey = NnFieldKey("url")
  val prefixFKey = NnFieldKey("prefix")
  val aliasFKey = NnFieldKey("alias")
  val graphFKey = NnFieldKey("graph")

  implicit class LegalChar(ch: Char) {
    // format: off
    def isLegal: Boolean = {
      val i = ch.toInt
      (45 to 57).contains(i)  ||     // (-, ., /, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
      (65 to 90).contains(i)  ||     // (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z)
      (97 to 122).contains(i) ||     // (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z)
      i == 35 || i == 37 || i == 44 || i == 58 || i == 95 || i > 126  // 35 = '#', 37 = '%', 44 = ',', 58 = ':', 95 = '_', 123 to 126 are: ({, |, }, ~)
    }
    // format: on
  }

  def createFieldValue(l: Literal, q: Option[String]): FieldValue = {
    val lang = {
      val lng = l.getLanguage
      if (lng.isEmpty) None
      else Some(lng)
{'fraction_non_alphanumeric': 0.14012196851510425, 'fraction_numerical': 0.01928804424904269, 'mean_word_length': 3.126389701579871, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '248221', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2445, 'n_words': 729}