<commit_before># Copyright (C) <NAME>. All rights reserved.
# BSD License. Look at license.txt for more info.
import parsecfg, json, streams, strutils, parseutils, os
import version, common
type
  TPackageInfo* = object
    mypath*: string ## The path of this .babel file
    name*: string
    version*: string
    author*: string
    description*: string
    license*: string
    skipDirs*: seq[string]
    skipFiles*: seq[string]
    skipExt*: seq[string]
    installDirs*: seq[string]
    installFiles*: seq[string]
    installExt*: seq[string]
    requires*: seq[tuple[name: string, ver: PVersionRange]]
    bin*: seq[string]
    srcDir*: string
    backend*: string

  TPackage* = object
    # Required fields in a package.
    name*: string
    url*: string # Download location.
    license*: string
    downloadMethod*: string
    description*: string
    tags*: seq[string] # Even if empty, always a valid non nil seq. \
    # From here on, optional fields set to the emtpy string if not available.
    version*: string
    dvcsTag*: string
    web*: string # Info url for humans.

proc initPackageInfo(): TPackageInfo =
  result.mypath = ""
  result.name = ""
  result.version = ""
  result.author = ""
  result.description = ""
  result.license = ""
  result.skipDirs = @[]
  result.skipFiles = @[]
  result.skipExt = @[]
  result.installDirs = @[]
  result.installFiles = @[]
  result.installExt = @[]
  result.requires = @[]
  result.bin = @[]
  result.srcDir = ""
  result.backend = "c"

proc validatePackageInfo(pkgInfo: TPackageInfo, path: string) =
  if pkgInfo.name == "":
    quit("Incorrect .babel file: " & path & " does not contain a name field.")
  if pkgInfo.version == "":
    quit("Incorrect .babel file: " & path & " does not contain a version field.")
  if pkgInfo.author == "":
    quit("Incorrect .babel file: " & path & " does not contain an author field.")
  if pkgInfo.description == "":
    quit("Incorrect .babel file: " & path & " does not contain a description field.")
  if pkgInfo.license == "":
    quit("Incorrect .babel file: " & path & " does not contain a license field.")
  if pkgInfo.backend notin ["c", "cc", "objc", "cpp", "js"]:
    raise newException(EBabel, "'" & pkgInfo.backend & "' is an invalid backend.")

proc parseRequires(req: string): tuple[name: string, ver: PVersionRange] =
  try:
    if ' ' in req:
      var i = skipUntil(req, whitespace)
      result.name = req[0 .. i].strip
      result.ver = parseVersionRange(req[i .. -1])
    else:
      result.name = req.strip
      result.ver = PVersionRange(kind: verAny)
  except EParseVersion:
    quit("Unable to parse dependency version range: " & getCurrentExceptionMsg())

proc readPackageInfo*(path: string): TPackageInfo =
  result = initPackageInfo()
  result.mypath = path
  var fs = newFileStream(path, fmRead)
  if fs != nil:
    var p: TCfgParser
    open(p, fs, path)
    var currentSection = ""
    while true:
      var ev = next(p)
      case ev.kind
      of cfgEof:
        break
      of cfgSectionStart:
        currentSection = ev.section
      of cfgKeyValuePair:
        case currentSection.normalize
        of "package":
          case ev.key.normalize
          of "name": result.name = ev.value
          of "version": result.version = ev.value
          of "author": result.author = ev.value
          of "description": result.description = ev.value
          of "license": result.license = ev.value
          of "srcdir": result.srcDir = ev.value
          of "skipdirs":
            result.skipDirs.add(ev.value.split(','))
          of "skipfiles":
            result.skipFiles.add(ev.value.split(','))
          of "skipext":
            result.skipExt.add(ev.value.split(','))
          of "installdirs":
            result.installDirs.add(ev.value.split(','))
          of "installfiles":
            result.installFiles.add(ev.value.split(','))
          of "installext":
            result.installExt.add(ev.value.split(','))
          of "bin":
            for i in ev.value.split(','):
              result.bin.add(i.addFileExt(ExeExt))
          of "backend":
            result.backend = ev.value.toLower()
            case result.backend.normalize
            of "javascript": result.backend = "js"
          else:
            quit("Invalid field: " & ev.key, QuitFailure)
        of "deps", "dependencies":
          case ev.key.normalize
          of "requires":
            for v in ev.value.split(','):
              result.requires.add(parseRequires(v.strip))
          else:
            quit("Invalid field: " & ev.key, QuitFailure)
        else: quit("Invalid section: " & currentSection, QuitFailure)
      of cfgOption: quit("Invalid package info, should not contain --" & ev.value, QuitFailure)
      of cfgError:
        echo(ev.msg)
    close(p)
  else:
    raise newException(EInvalidValue, "Cannot open package info: " & path)
  validatePackageInfo(result, path)

proc optionalField(obj: PJsonNode, name: string): string =
  if existsKey(obj, name):
    if obj[name].kind == JString:
      return obj[name].str
    else:
      quit("Corrupted packages.json file. " & name & " field is of unexpected type.")
  else: return ""

proc requiredField(obj: PJsonNode, name: string): string =
  if existsKey(obj, name):
    if obj[name].kind == JString:
      return obj[name].str
    else:
      quit("Corrupted packages.json file. " & name & " field is of unexpected type.")
  else:
    quit("Package in packages.json file does not contain a " & name & " field.")

proc getPackage*(pkg: string, packagesPath: string, resPkg: var TPackage): bool =
  let packages = parseFile(packagesPath)
  for p in packages:
    if p["name"].str != pkg: continue
    resPkg.name = pkg
    resPkg.url = p.requiredField("url")
    resPkg.version = p.optionalField("version")
    resPkg.downloadMethod = p.requiredField("method")
    resPkg.dvcsTag = p.optionalField("dvcs-tag")
    resPkg.license = p.requiredField("license")<commit_msg>Rewrites requiredField on top of optionalField.<commit_after># Copyright (C) <NAME>. All rights reserved.
# BSD License. Look at license.txt for more info.
import parsecfg, json, streams, strutils, parseutils, os
import version, common
type
  TPackageInfo* = object
    mypath*: string ## The path of this .babel file
    name*: string
    version*: string
    author*: string
    description*: string
    license*: string
    skipDirs*: seq[string]
    skipFiles*: seq[string]
    skipExt*: seq[string]
    installDirs*: seq[string]
    installFiles*: seq[string]
    installExt*: seq[string]
    requires*: seq[tuple[name: string, ver: PVersionRange]]
    bin*: seq[string]
    srcDir*: string
    backend*: string

  TPackage* = object
    # Required fields in a package.
    name*: string
    url*: string # Download location.
    license*: string
    downloadMethod*: string
    description*: string
    tags*: seq[string] # Even if empty, always a valid non nil seq. \
    # From here on, optional fields set to the emtpy string if not available.
    version*: string
    dvcsTag*: string
    web*: string # Info url for humans.

proc initPackageInfo(): TPackageInfo =
  result.mypath = ""
  result.name = ""
  result.version = ""
  result.author = ""
  result.description = ""
  result.license = ""
  result.skipDirs = @[]
  result.skipFiles = @[]
  result.skipExt = @[]
  result.installDirs = @[]
  result.installFiles = @[]
  result.installExt = @[]
  result.requires = @[]
  result.bin = @[]
  result.srcDir = ""
  result.backend = "c"

proc validatePackageInfo(pkgInfo: TPackageInfo, path: string) =
  if pkgInfo.name == "":
    quit("Incorrect .babel file: " & path & " does not contain a name field.")
  if pkgInfo.version == "":
    quit("Incorrect .babel file: " & path & " does not contain a version field.")
  if pkgInfo.author == "":
    quit("Incorrect .babel file: " & path & " does not contain an author field.")
  if pkgInfo.description == "":
    quit("Incorrect .babel file: " & path & " does not contain a description field.")
  if pkgInfo.license == "":
    quit("Incorrect .babel file: " & path & " does not contain a license field.")
  if pkgInfo.backend notin ["c", "cc", "objc", "cpp", "js"]:
    raise newException(EBabel, "'" & pkgInfo.backend & "' is an invalid backend.")

proc parseRequires(req: string): tuple[name: string, ver: PVersionRange] =
  try:
    if ' ' in req:
      var i = skipUntil(req, whitespace)
      result.name = req[0 .. i].strip
      result.ver = parseVersionRange(req[i .. -1])
    else:
      result.name = req.strip
      result.ver = PVersionRange(kind: verAny)
  except EParseVersion:
    quit("Unable to parse dependency version range: " & getCurrentExceptionMsg())

proc readPackageInfo*(path: string): TPackageInfo =
  result = initPackageInfo()
  result.mypath = path
  var fs = newFileStream(path, fmRead)
  if fs != nil:
    var p: TCfgParser
    open(p, fs, path)
    var currentSection = ""
    while true:
      var ev = next(p)
      case ev.kind
      of cfgEof:
        break
      of cfgSectionStart:
        currentSection = ev.section
      of cfgKeyValuePair:
        case currentSection.normalize
        of "package":
          case ev.key.normalize
          of "name": result.name = ev.value
          of "version": result.version = ev.value
          of "author": result.author = ev.value
          of "description": result.description = ev.value
          of "license": result.license = ev.value
          of "srcdir": result.srcDir = ev.value
          of "skipdirs":
            result.skipDirs.add(ev.value.split(','))
          of "skipfiles":
            result.skipFiles.add(ev.value.split(','))
          of "skipext":
            result.skipExt.add(ev.value.split(','))
          of "installdirs":
            result.installDirs.add(ev.value.split(','))
          of "installfiles":
            result.installFiles.add(ev.value.split(','))
          of "installext":
            result.installExt.add(ev.value.split(','))
          of "bin":
            for i in ev.value.split(','):
              result.bin.add(i.addFileExt(ExeExt))
          of "backend":
            result.backend = ev.value.toLower()
            case result.backend.normalize
            of "javascript": result.backend = "js"
          else:
            quit("Invalid field: " & ev.key, QuitFailure)
        of "deps", "dependencies":
          case ev.key.normalize
          of "requires":
            for v in ev.value.split(','):
              result.requires.add(parseRequires(v.strip))
          else:
            quit("Invalid field: " & ev.key, QuitFailure)
        else: quit("Invalid section: " & currentSection, QuitFailure)
      of cfgOption: quit("Invalid package info, should not contain --" & ev.value, QuitFailure)
      of cfgError:
        echo(ev.msg)
    close(p)
  else:
    raise newException(EInvalidValue, "Cannot open package info: " & path)
  validatePackageInfo(result, path)

proc optionalField(obj: PJsonNode, name: string, default = ""): string =
  ## Queries ``obj`` for the optional ``name`` string.
  ##
  ## Returns the value of ``name`` if it is a valid string, or aborts execution
  ## if the field exists but is not of string type. If ``name`` is not present,
  ## returns ``default``.
  if existsKey(obj, name):
    if obj[name].kind == JString:
      return obj[name].str
    else:
      quit("Corrupted packages.json file. " & name & " field is of unexpected type.")
  else: return default

proc requiredField(obj: PJsonNode, name: string): string =
  ## Queries ``obj`` for the required ``name`` string.
  ##
  ## Aborts execution if the field does not exist or is of invalid json type.
  result = optionalField(obj, name, nil)
  if result == nil:
    quit("Package in packages.json file does not contain a " & name & " field.")

proc getPackage*(pkg: string, packagesPath: string, resPkg: var TPackage): bool =
  let packages = parseFile(packagesPath)
  for p in packages:
    if p["name"].str != pkg: continue
    resPkg.name = pkg
    resPkg.url = p.requiredField("url")
    resPkg.version = p.optionalField("version")
    resPkg.downloadMethod = p.requiredField("method")
    resPkg.dvcsTag = p.optionalField("dvcs-tag")
    resPkg.license = p.requiredField("license")
{'fraction_non_alphanumeric': 0.13003752651329745, 'fraction_numerical': 0.00032631750693424703, 'mean_word_length': 2.714848484848485, 'pattern_counts': {'":': 46, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2706123', 'n_tokens_mistral': 3711, 'n_tokens_neox': 3667, 'n_words': 1067}