<commit_before>const {CONTEXT, GRAPH, ID, TYPE, VALUE, LANGUAGE, LIST, VOCAB, BASE} =
  require('../jsonld/keywords')

module.exports = {serialize}

function serialize (data, out = process.stdout, {
  context = null,
  baseIri = null,
  settings = {
    indentChars: '  ',
    upcaseKeywords: false,
    useGraphKeyword: true,
    spaceBeforeSemicolon: true,
    predicateRepeatNewLine: true,
    bracketStartNewLine: false,
    bracketEndNewLine: false,
    listItemNewLine: true,
    prologueEndLine: 1,
    separatorLines: 1
  }
} = {}) {
  let state = new SerializerState(context, out, settings)
  state.serialize(data)
}

class SerializerState {
  constructor (context, outstream, settings, parent = null) {
    this.outstream = outstream || parent.outstream
    this.initContext(context)
    this.parent = parent
    this.settings = parent ? parent.settings : settings
    let kw = it => this.settings.upcaseKeywords ? it.toUpperCase() : it
    this.prefixKeyword = kw('prefix')
    this.baseKeyword = kw('base')
    if (this.settings.useGraphKeyword) {
      this.graphKeyword = kw('graph')
    }
    this.uniqueBNodeSuffix = ''
  }

  initContext (context) {
    this.keys = {
      id: "@id",
      value: '@value',
      type: '@type',
      lang: '@language',
      graph: '@graph',
      list: '@list'
    }
    this.prefixes = {}

    if (context == null) {
      this.context = {}
    }
    else {
      for (let key in this.context) {
        throw 'context already initialized'
      }
    }
    let ctx = {}
    // TODO: merge arrays, deref uri:s ...
    if (Array.isArray(context)) {
      for (let item of context) {
        Object.assign(ctx, item)
      }
    } else if (typeof context === 'object') {
        Object.assign(ctx, context)
    }
    this.context = ctx
    this.prefixes = collectPrefixes(context)
  }

  serialize (data) {
    if (data[CONTEXT]) {
      this.initContext(data[CONTEXT])
    }
    this.prelude()
    let graph = Array.isArray(data) ? data : data[GRAPH]
    if (graph) {
      for (let node of graph) {
        this.objectToTurtle(node)
      }
    } else {
      this.objectToTurtle(data)
    }
  }

  prelude () {
    for (let k in this.prefixes) {
      let v = this.prefixes[k]
      if (k === BASE) {
        this.writeBase(v)
      } else {
        this.writeln(`${this.prefixKeyword} ${k}: <${v}>`)
      }
    }
    if (this.base) {
      this.writeBase(this.base)
    }
    if (this.settings.prologueEndLine > 1) {
      this.writeln()
    }
  }

  writeBase (iri) {
    this.writeln(`${this.baseKeyword} <${iri}>`)
  }

  isListContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@list'
  }

  isLangContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@language'
  }

  objectToTrig (iri, graph) {
    this.writeln()
    if (this.graphKeyword) {
        this.write(`${this.graphKeyword} `)
    }
    this.writeln(`${this.refRepr(iri)} {`)
    if (!Array.isArray(graph)) graph = [graph]
    for (let node of graph) {
      this.objectToTurtle(node, 0, GRAPH)
    }
    this.writeln()
    this.writeln("}")
  }

  objectToTurtle (obj, depth = 0, viaKey = null) {
    if (depth > 0 && obj[CONTEXT]) {
      // TODO: use new SerializerState(node[CONTEXT], null, null, state)
      throw 'Nested context not supported yet'
    }

    if (this.isLangContainer(viaKey) && typeof obj === 'object') {
      let first = true
      for (let lang in obj) {
        let value = obj[lang]
          if (!first) this.write(' , ')
        this.toLiteral(
          { [this.keys.value]: value, [this.keys.lang]: lang },
          viaKey)
        first = false
      }
      return []
    }

    if (!(typeof obj === 'object') || obj[this.keys.value]) {
      this.toLiteral(obj, viaKey)
      return []
    }

    let explicitList = obj[LIST]

    if (this.isListContainer(viaKey)) {
      obj = { [this.keys.list]: obj }
    }

    let s = obj[this.keys.id]

    let isList = obj[LIST]
    let startedList = isList

    if (obj[this.keys.graph]) {
      this.objectToTrig(s, obj[this.keys.graph])
      return []
    }

    if (explicitList) {
      this.write('( ')
    }

    let inGraph = viaKey === GRAPH ? 1 : 0

    if (s != null && this.hasKeys(obj, 2)) {
      if (depth === 0) {
        this.writeln()
      }
      if (inGraph) this.write(this.getIndent(0))
      this.write(this.refRepr(s))
    } else if (depth > 0) {
      if (!isList) {
        depth++
        this.write("[")
      }
    } else {
      if (obj[this.keys.graph]) {
        /* TODO:
        for (let key in obj) {
          if (key[0] != '@') {
            // TODO: this is the default graph, described...
            this.write('[]')
            break
          }
        }
        */
      } else {
        return []
      }
    }

    let indent = this.getIndent(depth + inGraph)

    let topObjects = []

    let first = true
    let endedList = false

    for (let key in obj) {
      let vs = obj[key]
      let term = this.termFor(key)

      let revKey = (term == null)? this.revKeyFor(key) : null
      if (term == null && revKey == null)
        continue

      if (term == this.keys.id || term == "@context")
        continue

      vs = Array.isArray(vs) ? vs : vs != null ? [vs] : []
      if (!vs.length) // TODO: && not @list
        continue

      let inList = isList || this.isListContainer(key)

      if (revKey) {
        for (let it of vs) {
          let node = it.clone()
          node[revKey] = { [this.keys.id]: s }
          topObjects.push(node)
        }
      } else {
        let useIndent = indent
        if (first) {
          useIndent = ' '
          first = false
        } else {
          if (startedList && !inList && !endedList) {
            endedList = true
            this.write(" )")
          }
          this.writeln(" ;")
        }

        if (term == "@type") {
          term = "a"
          this.write(useIndent + term + " " + vs.map(it =>
            this.toValidTerm(this.termFor(it))
          ).join(", "))
          continue
        }

        if (term != '@list') {
          term = this.toValidTerm(term)
          this.write(useIndent + term + " ")
        }

        for (let i = 0; i < vs.length; i++) {
          let v = vs[i]
            if (inList) {
              if (!startedList) {
                this.write("(")
                startedList = true
              }
              this.write(" ")
            } else if (i > 0) {
              if (this.settings.predicateRepeatNewLine) {
                this.writeln(' ,')
                this.write(this.getIndent(depth + 1 + inGraph))
              } else {
                this.write(' , ')
              }
            }
          if (this.bnodeSkolemBase && typeof v === 'object' && !v[this.keys.id]) {
            v[this.keys.id] = s = this.genSkolemId()
          }
          if (typeof v === 'object' && this.keys.id in v) {
            topObjects.push(v)
            this.write(this.refRepr(v[this.keys.id]))
          } else {
            for (let it of this.objectToTurtle(v, depth + 1, key)) {
              topObjects.push(it)
            }
          }
        }
      }
    }

    if (explicitList || (!isList && startedList) && !endedList) {
      this.write(" )")
    }

    if (depth == 0) {
      if (!first) {
        this.writeln(" .")
      }
      //this.writeln()
      for (let it of topObjects) {
        this.objectToTurtle(it)
      }
      return []
    } else {
      indent = this.getIndent(depth - 1 + inGraph)
      if (this.settings.bracketEndNewLine) {
        this.writeln()
        this.write(indent)
      } else {
        this.write(' ')
      }
      if (!isList) {
        // NOTE: hack for e.g. BlazeGraph
        if (!this.hasKeys(obj) && markEmptyBnode) {
          this.writeln("a $emptyMarker")
          this.write(indent)
        }
        this.write("]")
      }
      return topObjects
    }
  }

  toLiteral (obj, viaKey=null) {
    let value = obj
    let lang = this.context["@language"]
    let datatype = null
    if (typeof obj === 'object') {
      value = obj[this.keys.value]
      datatype = obj[this.keys.type]
      lang = obj[this.keys.lang]
    } else {
      let kdef = this.context[viaKey]
      let coerceTo = (typeof kdef === 'object')? kdef["@type"] : null
      if (coerceTo == "@vocab") {
        this.write(typeof value === 'string' ?
          this.refRepr(value, true): value)
        return
      } else if (coerceTo == "@id") {
        this.write(this.refRepr(value))
        return
      } else if (coerceTo) {
        datatype = coerceTo
      } else {
        let termLang = (typeof kdef === 'object')? kdef["@language"] : null
        if (typeof kdef === 'object') {
          lang = termLang
        }
      }
    }
    if (typeof value === 'string') {
      let escaped = value.replace(/"/g, '\\"')
      let quote = '"'
      if (escaped.indexOf('\n') > -1) {
        quote = '"""'
      }
      this.write(quote)
      this.write(escaped)
      this.write(quote)
      if (datatype)
        this.write("^^" + this.termFor(datatype))
      else if (lang)
        this.write("@" + lang)
    } else { // boolean or number
      this.write(value.toString())
    }
  }

  termFor (key) {
    if (key.startsWith("@")) {
      return key
    } else if (key.indexOf(":") > -1) {
      return key
    } else if (this.context[key]) {
      let kdef = this.context[key]
      if (kdef == null)
        return null
      let term = null
      if (typeof kdef === 'object') {
        term = kdef["@id"] || key
      } else {
        term = kdef
      }
      if (term == null)
        return null
      return (term.indexOf(":") == -1)? ":" + term : term
    } else {
      return ":" + key
    }
  }

  revKeyFor (key) {
    let kdef = this.context[key]
    if (!(typeof kdef === 'object'))
      return null
    return kdef["@reverse"]
  }

  refRepr (ref, useVocab = false) {
    let cI = ref.indexOf(":")
    if (cI > -1) {
      let pfx = ref.substring(0, cI)
      if (pfx == "_") {
        let nodeId = ref + this.uniqueBNodeSuffix
        if (this.bnodeSkolemBase) {
          ref = this.bnodeSkolemBase + nodeId.substring(2)
        } else {
          return this.toValidTerm(nodeId)
        }
      } else if (this.context[pfx]) {
        return ref
      }
    } else if (useVocab && ref.indexOf("/") == -1) {
      return ":" + ref
    }

    if (ref.startsWith(this.context[VOCAB])) {
      return ":" + ref.substring(this.context[VOCAB].length)
    }

    ref = this.cleanValue(ref)

    return `<${ref}>`
  }
<commit_msg>Represent missing id as blank node<commit_after>const {CONTEXT, GRAPH, ID, TYPE, VALUE, LANGUAGE, LIST, VOCAB, BASE} =
  require('../jsonld/keywords')

module.exports = {serialize}

function serialize (data, out = process.stdout, {
  context = null,
  baseIri = null,
  settings = {
    indentChars: '  ',
    upcaseKeywords: false,
    useGraphKeyword: true,
    spaceBeforeSemicolon: true,
    predicateRepeatNewLine: true,
    bracketStartNewLine: false,
    bracketEndNewLine: false,
    listItemNewLine: true,
    prologueEndLine: 1,
    separatorLines: 1
  }
} = {}) {
  let state = new SerializerState(context, out, settings)
  state.serialize(data)
}

class SerializerState {
  constructor (context, outstream, settings, parent = null) {
    this.outstream = outstream || parent.outstream
    this.initContext(context)
    this.parent = parent
    this.settings = parent ? parent.settings : settings
    let kw = it => this.settings.upcaseKeywords ? it.toUpperCase() : it
    this.prefixKeyword = kw('prefix')
    this.baseKeyword = kw('base')
    if (this.settings.useGraphKeyword) {
      this.graphKeyword = kw('graph')
    }
    this.uniqueBNodeSuffix = ''
  }

  initContext (context) {
    this.keys = {
      id: "@id",
      value: '@value',
      type: '@type',
      lang: '@language',
      graph: '@graph',
      list: '@list'
    }
    this.prefixes = {}

    if (context == null) {
      this.context = {}
    }
    else {
      for (let key in this.context) {
        throw 'context already initialized'
      }
    }
    let ctx = {}
    // TODO: merge arrays, deref uri:s ...
    if (Array.isArray(context)) {
      for (let item of context) {
        Object.assign(ctx, item)
      }
    } else if (typeof context === 'object') {
        Object.assign(ctx, context)
    }
    this.context = ctx
    this.prefixes = collectPrefixes(context)
  }

  serialize (data) {
    if (data[CONTEXT]) {
      this.initContext(data[CONTEXT])
    }
    this.prelude()
    let graph = Array.isArray(data) ? data : data[GRAPH]
    if (graph) {
      for (let node of graph) {
        this.objectToTurtle(node)
      }
    } else {
      this.objectToTurtle(data)
    }
  }

  prelude () {
    for (let k in this.prefixes) {
      let v = this.prefixes[k]
      if (k === BASE) {
        this.writeBase(v)
      } else {
        this.writeln(`${this.prefixKeyword} ${k}: <${v}>`)
      }
    }
    if (this.base) {
      this.writeBase(this.base)
    }
    if (this.settings.prologueEndLine > 1) {
      this.writeln()
    }
  }

  writeBase (iri) {
    this.writeln(`${this.baseKeyword} <${iri}>`)
  }

  isListContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@list'
  }

  isLangContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@language'
  }

  objectToTrig (iri, graph) {
    this.writeln()
    if (this.graphKeyword) {
        this.write(`${this.graphKeyword} `)
    }
    this.writeln(`${this.refRepr(iri)} {`)
    if (!Array.isArray(graph)) graph = [graph]
    for (let node of graph) {
      this.objectToTurtle(node, 0, GRAPH)
    }
    this.writeln()
    this.writeln("}")
  }

  objectToTurtle (obj, depth = 0, viaKey = null) {
    if (depth > 0 && obj[CONTEXT]) {
      // TODO: use new SerializerState(node[CONTEXT], null, null, state)
      throw 'Nested context not supported yet'
    }

    if (this.isLangContainer(viaKey) && typeof obj === 'object') {
      let first = true
      for (let lang in obj) {
        let value = obj[lang]
          if (!first) this.write(' , ')
        this.toLiteral(
          { [this.keys.value]: value, [this.keys.lang]: lang },
          viaKey)
        first = false
      }
      return []
    }

    if (!(typeof obj === 'object') || obj[this.keys.value]) {
      this.toLiteral(obj, viaKey)
      return []
    }

    let explicitList = obj[LIST]

    if (this.isListContainer(viaKey)) {
      obj = { [this.keys.list]: obj }
    }

    let s = obj[this.keys.id]

    let isList = obj[LIST]
    let startedList = isList

    if (obj[this.keys.graph]) {
      this.objectToTrig(s, obj[this.keys.graph])
      return []
    }

    if (explicitList) {
      this.write('( ')
    }

    let inGraph = viaKey === GRAPH ? 1 : 0

    if (s != null && this.hasKeys(obj, 2)) {
      if (depth === 0) {
        this.writeln()
      }
      if (inGraph) this.write(this.getIndent(0))
      this.write(this.refRepr(s))
    } else if (depth > 0) {
      if (!isList) {
        depth++
        this.write("[")
      }
    } else {
      if (obj[this.keys.graph]) {
        /* TODO:
        for (let key in obj) {
          if (key[0] != '@') {
            // TODO: this is the default graph, described...
            this.write('[]')
            break
          }
        }
        */
      } else {
        return []
      }
    }

    let indent = this.getIndent(depth + inGraph)

    let topObjects = []

    let first = true
    let endedList = false

    for (let key in obj) {
      let vs = obj[key]
      let term = this.termFor(key)

      let revKey = (term == null)? this.revKeyFor(key) : null
      if (term == null && revKey == null)
        continue

      if (term == this.keys.id || term == "@context")
        continue

      vs = Array.isArray(vs) ? vs : vs != null ? [vs] : []
      if (!vs.length) // TODO: && not @list
        continue

      let inList = isList || this.isListContainer(key)

      if (revKey) {
        for (let it of vs) {
          let node = it.clone()
          node[revKey] = { [this.keys.id]: s }
          topObjects.push(node)
        }
      } else {
        let useIndent = indent
        if (first) {
          useIndent = ' '
          first = false
        } else {
          if (startedList && !inList && !endedList) {
            endedList = true
            this.write(" )")
          }
          this.writeln(" ;")
        }

        if (term == "@type") {
          term = "a"
          this.write(useIndent + term + " " + vs.map(it =>
            this.toValidTerm(this.termFor(it))
          ).join(", "))
          continue
        }

        if (term != '@list') {
          term = this.toValidTerm(term)
          this.write(useIndent + term + " ")
        }

        for (let i = 0; i < vs.length; i++) {
          let v = vs[i]
            if (inList) {
              if (!startedList) {
                this.write("(")
                startedList = true
              }
              this.write(" ")
            } else if (i > 0) {
              if (this.settings.predicateRepeatNewLine) {
                this.writeln(' ,')
                this.write(this.getIndent(depth + 1 + inGraph))
              } else {
                this.write(' , ')
              }
            }
          if (this.bnodeSkolemBase && typeof v === 'object' && !v[this.keys.id]) {
            v[this.keys.id] = s = this.genSkolemId()
          }
          if (typeof v === 'object' && this.keys.id in v) {
            topObjects.push(v)
            this.write(this.refRepr(v[this.keys.id]))
          } else {
            for (let it of this.objectToTurtle(v, depth + 1, key)) {
              topObjects.push(it)
            }
          }
        }
      }
    }

    if (explicitList || (!isList && startedList) && !endedList) {
      this.write(" )")
    }

    if (depth == 0) {
      if (!first) {
        this.writeln(" .")
      }
      //this.writeln()
      for (let it of topObjects) {
        this.objectToTurtle(it)
      }
      return []
    } else {
      indent = this.getIndent(depth - 1 + inGraph)
      if (this.settings.bracketEndNewLine) {
        this.writeln()
        this.write(indent)
      } else {
        this.write(' ')
      }
      if (!isList) {
        // NOTE: hack for e.g. BlazeGraph
        if (!this.hasKeys(obj) && markEmptyBnode) {
          this.writeln("a $emptyMarker")
          this.write(indent)
        }
        this.write("]")
      }
      return topObjects
    }
  }

  toLiteral (obj, viaKey=null) {
    let value = obj
    let lang = this.context["@language"]
    let datatype = null
    if (typeof obj === 'object') {
      value = obj[this.keys.value]
      datatype = obj[this.keys.type]
      lang = obj[this.keys.lang]
    } else {
      let kdef = this.context[viaKey]
      let coerceTo = (typeof kdef === 'object')? kdef["@type"] : null
      if (coerceTo == "@vocab") {
        this.write(typeof value === 'string' ?
          this.refRepr(value, true): value)
        return
      } else if (coerceTo == "@id") {
        this.write(this.refRepr(value))
        return
      } else if (coerceTo) {
        datatype = coerceTo
      } else {
        let termLang = (typeof kdef === 'object')? kdef["@language"] : null
        if (typeof kdef === 'object') {
          lang = termLang
        }
      }
    }
    if (typeof value === 'string') {
      let escaped = value.replace(/"/g, '\\"')
      let quote = '"'
      if (escaped.indexOf('\n') > -1) {
        quote = '"""'
      }
      this.write(quote)
      this.write(escaped)
      this.write(quote)
      if (datatype)
        this.write("^^" + this.termFor(datatype))
      else if (lang)
        this.write("@" + lang)
    } else { // boolean or number
      this.write(value.toString())
    }
  }

  termFor (key) {
    if (key.startsWith("@")) {
      return key
    } else if (key.indexOf(":") > -1) {
      return key
    } else if (this.context[key]) {
      let kdef = this.context[key]
      if (kdef == null)
        return null
      let term = null
      if (typeof kdef === 'object') {
        term = kdef["@id"] || key
      } else {
        term = kdef
      }
      if (term == null)
        return null
      return (term.indexOf(":") == -1)? ":" + term : term
    } else {
      return ":" + key
    }
  }

  revKeyFor (key) {
    let kdef = this.context[key]
    if (!(typeof kdef === 'object'))
      return null
    return kdef["@reverse"]
  }

  refRepr (ref, useVocab = false) {
    if (ref == null) return '[]'

    let cI = ref.indexOf(":")
    if (cI > -1) {
      let pfx = ref.substring(0, cI)
      if (pfx == "_") {
        let nodeId = ref + this.uniqueBNodeSuffix
        if (this.bnodeSkolemBase) {
          ref = this.bnodeSkolemBase + nodeId.substring(2)
        } else {
          return this.toValidTerm(nodeId)
        }
      } else if (this.context[pfx]) {
        return ref
      }
    } else if (useVocab && ref.indexOf("/") == -1) {
      return ":" + ref
    }

    if (ref.startsWith(this.context[VOCAB])) {
      return ":" + ref.substring(this.context[VOCAB].length)
    }

    ref = this.cleanValue(ref)

    return `<${ref}>`
  }

{'fraction_non_alphanumeric': 0.15261230355569977, 'fraction_numerical': 0.0024106439200778824, 'mean_word_length': 1.7802551875241654, 'pattern_counts': {'":': 14, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '883601', 'n_tokens_mistral': 7246, 'n_tokens_neox': 7117, 'n_words': 1587}