<commit_before>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package laika.parse.util

import laika.parse.MarkupParsers

/**
 * Parses URIs as defined in RFC 3986 and email addresses as defined in
 * RFC 6068, 5322 and 3986 with base types defined in RFC 2234. 
 * 
 * @author <NAME>
 */
trait URIParsers extends MarkupParsers {

  
  /** Flattens the result from various combinators,
   *  including the `repX` variants and `~` into 
   *  a single string.
   */
  def flatten (result: Any): String = result match {
    case Some(x)      => flatten(x)
    case None         => ""
    case c: Char      => c.toString
    case s: String    => s
    case l: List[Any] => l.map(flatten).mkString
    case x ~ y        => flatten(x) + flatten(y)
  }
  

  /** Parses letters according to RFC 2234.
   * 
   *  {{{
   *  ALPHA =  %x41-5A / %x61-7A ; A-Z / a-z
   *  }}}
   */
  val alpha = anyIn('a' to 'z', 'A' to 'Z')

  /** Parses digits according to RFC 2234.
   * 
   *  {{{
   *  DIGIT =  %x30-39; 0-9
   *  }}}
   */
  val digit = anyIn('0' to '9')

  /** Parses a hexadecimal value according to RFC 2234.
   * 
   *  {{{
   *  HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
   *  }}}
   */
  val hexdig = anyIn('0' to '9', 'A' to 'F')<commit_msg>avoid unchecked warning in URIParsers<commit_after>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package laika.parse.util

import laika.parse.MarkupParsers

/**
 * Parses URIs as defined in RFC 3986 and email addresses as defined in
 * RFC 6068, 5322 and 3986 with base types defined in RFC 2234. 
 * 
 * @author <NAME>
 */
trait URIParsers extends MarkupParsers {

  
  /** Flattens the result from various combinators,
   *  including the `repX` variants and `~` into 
   *  a single string.
   */
  def flatten (result: Any): String = result match {
    case Some(x)      => flatten(x)
    case None         => ""
    case c: Char      => c.toString
    case s: String    => s
    case l: List[_]   => l.map(flatten).mkString
    case x ~ y        => flatten(x) + flatten(y)
  }
  

  /** Parses letters according to RFC 2234.
   * 
   *  {{{
   *  ALPHA =  %x41-5A / %x61-7A ; A-Z / a-z
   *  }}}
   */
  val alpha = anyIn('a' to 'z', 'A' to 'Z')

  /** Parses digits according to RFC 2234.
   * 
   *  {{{
   *  DIGIT =  %x30-39; 0-9
   *  }}}
   */
  val digit = anyIn('0' to '9')

  /** Parses a hexadecimal value according to RFC 2234.
   * 
   *  {{{
   *  HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
   *  }}}
   */
  val hexdig = anyIn('0' to '9', 'A' to 'F')
{'fraction_non_alphanumeric': 0.12636165577342048, 'fraction_numerical': 0.030501089324618737, 'mean_word_length': 2.448826291079812, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4251000', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1228, 'n_words': 447}