<commit_before>require 'json/add/core'
require 'json/pure'
require 'couchrest'
require 'metainspector'

class Parsed #The dreaded CSS selector class! Why did KVR have to use images?!
  def win(pdoc)
    pdoc.at_css(".sp1b")
  end
  def win64(pdoc)
    pdoc.at_css(".sp2b")
  end
  def mac(pdoc)
    pdoc.at_css(".sp3b")
  end
  def mac64(pdoc)
    pdoc.at_css(".sp6b")
  end
  def lin(pdoc)
    pdoc.at_css(".sp7b")
  end
  def ios(pdoc)
    pdoc.at_css(".sp14b")
  end
  def droid(pdoc)
    pdoc.at_css(".sp15b")
  end
  def vst(pdoc)
    pdoc.at_css(".sp20b")
  end
  def vst3(pdoc)
    pdoc.at_css(".sp21b")
  end
  def au(pdoc)
    pdoc.at_css(".sp22b")
  end
  def dx(pdoc)
    pdoc.at_css(".sp23b")
  end
  def rtas(pdoc)
    pdoc.at_css(".sp24b")
  end
  def ladspa(pdoc)
    pdoc.at_css(".sp25b")
  end
  def lv2(pdoc)
    pdoc.at_css(".sp26b")
  end
  def rewire(pdoc)
    pdoc.at_css(".sp27b")
  end
  def aax(pdoc)
    pdoc.at_css(".sp29b")
  end
  def dssi(pdoc)
    pdoc.at_css(".sp30b")
  end
  def re(pdoc)
    pdoc.at_css(".sp31b")
  end
end
class Txt
  def inst(ptext)
    ptext =~ /Instrument\(s\)/
  end
  def eff(ptext)
    ptext =~ /Effect\(s\)/
  end
  def chinst(ptext)
    ptext =~ /Can HostInstruments/
  end
  def cheff(ptext)
    ptext =~ /Can HostEffects/
  end
end
class Pachinko #Ever played Pachinko?
  def type(pdoc, formats)
    parsed=Parsed.new
    if parsed.vst(pdoc) != nil then formats.push "VST"; end
    if parsed.vst3(pdoc) != nil then formats.push "VST3"; end
    if parsed.au(pdoc) != nil then formats.push "Audio Unit"; end
    if parsed.dx(pdoc) != nil then formats.push "DirectX"; end
    if parsed.rtas(pdoc) != nil then formats.push "RTAS"; end
    if parsed.ladspa(pdoc) != nil then formats.push "LADSPA"; end
    if parsed.lv2(pdoc) != nil then formats.push "LV2"; end
    if parsed.rewire(pdoc) != nil then formats.push "ReWire"; end
    if parsed.aax(pdoc) != nil then formats.push "AAX"; end
    if parsed.dssi(pdoc) != nil then formats.push "DSSI"; end
    if parsed.re(pdoc) != nil then formats.push "Rack Extension"; end
  end
  def os(pdoc, platforms)
    parsed=Parsed.new
    if parsed.win(pdoc) != nil then platforms.push "Windows"; end
    if parsed.win64(pdoc) != nil then platforms.push "Windows x64"; end
    if parsed.mac(pdoc) != nil then platforms.push "Mac OSX"; end
    if parsed.mac64(pdoc) != nil then platforms.push "Mac x64"; end
    if parsed.lin(pdoc) != nil then platforms.push "Linux"; end
    if parsed.ios(pdoc) != nil then platforms.push "iOS"; end
    if parsed.droid(pdoc) != nil then platforms.push "Android"; end
  end
  def function(ptext, capabilities)
    txt=Txt.new
    if txt.inst(ptext) != nil then capabilities.push "Instrument"; end
    if txt.eff(ptext) != nil then capabilities.push "Effect"; end
    if txt.chinst(ptext) != nil then capabilities.push "Instrument Host"; end
    if txt.cheff(ptext) != nil then capabilities.push "Effect Host"; end
  end
end
print 'Stage two.'
lnk='plinks.txt'
plinks=File.readlines(lnk)
x=0
y=plinks.length
y-=1 #zero to y
while x <= y
  filename="json/#{x}.json"
  if FileTest.exist?(filename) == true
    print 'x'
    x+=1
  else
    formats=[]
    capabilities=[]
    platforms=[]
    query=plinks[x].chomp
    kvr=MetaInspector.new(query) #this data needs alot of work
    pdoc=kvr.parsed
    ptext=pdoc.text
    if ptext =~ /product is no longer listed at KVR Audio/ || pdoc.at_css('#pluginsblock div p') == nil
      print 'x'
      x+=1
    else
      pachinko=Pachinko.new
      pachinko.type(pdoc, formats)
      pachinko.os(pdoc, platforms)
      pachinko.function(ptext, capabilities)
      hash=kvr.to_hash<commit_msg>fix build & fix json filenames<commit_after>require 'json/add/core'
require 'json/pure'
require 'couchrest'
require 'metainspector'

class Parsed #The dreaded CSS selector class! Why did KVR have to use images?!
  def win(pdoc)
    pdoc.at_css(".sp1b")
  end
  def win64(pdoc)
    pdoc.at_css(".sp2b")
  end
  def mac(pdoc)
    pdoc.at_css(".sp3b")
  end
  def mac64(pdoc)
    pdoc.at_css(".sp6b")
  end
  def lin(pdoc)
    pdoc.at_css(".sp7b")
  end
  def ios(pdoc)
    pdoc.at_css(".sp14b")
  end
  def droid(pdoc)
    pdoc.at_css(".sp15b")
  end
  def vst(pdoc)
    pdoc.at_css(".sp20b")
  end
  def vst3(pdoc)
    pdoc.at_css(".sp21b")
  end
  def au(pdoc)
    pdoc.at_css(".sp22b")
  end
  def dx(pdoc)
    pdoc.at_css(".sp23b")
  end
  def rtas(pdoc)
    pdoc.at_css(".sp24b")
  end
  def ladspa(pdoc)
    pdoc.at_css(".sp25b")
  end
  def lv2(pdoc)
    pdoc.at_css(".sp26b")
  end
  def rewire(pdoc)
    pdoc.at_css(".sp27b")
  end
  def aax(pdoc)
    pdoc.at_css(".sp29b")
  end
  def dssi(pdoc)
    pdoc.at_css(".sp30b")
  end
  def re(pdoc)
    pdoc.at_css(".sp31b")
  end
end
class Txt
  def inst(ptext)
    ptext =~ /Instrument\(s\)/
  end
  def eff(ptext)
    ptext =~ /Effect\(s\)/
  end
  def chinst(ptext)
    ptext =~ /Can HostInstruments/
  end
  def cheff(ptext)
    ptext =~ /Can HostEffects/
  end
end
class Pachinko #Ever played Pachinko?
  def type(pdoc, formats)
    parsed=Parsed.new
    if parsed.vst(pdoc) != nil then formats.push "VST"; end
    if parsed.vst3(pdoc) != nil then formats.push "VST3"; end
    if parsed.au(pdoc) != nil then formats.push "Audio Unit"; end
    if parsed.dx(pdoc) != nil then formats.push "DirectX"; end
    if parsed.rtas(pdoc) != nil then formats.push "RTAS"; end
    if parsed.ladspa(pdoc) != nil then formats.push "LADSPA"; end
    if parsed.lv2(pdoc) != nil then formats.push "LV2"; end
    if parsed.rewire(pdoc) != nil then formats.push "ReWire"; end
    if parsed.aax(pdoc) != nil then formats.push "AAX"; end
    if parsed.dssi(pdoc) != nil then formats.push "DSSI"; end
    if parsed.re(pdoc) != nil then formats.push "Rack Extension"; end
  end
  def os(pdoc, platforms)
    parsed=Parsed.new
    if parsed.win(pdoc) != nil then platforms.push "Windows"; end
    if parsed.win64(pdoc) != nil then platforms.push "Windows x64"; end
    if parsed.mac(pdoc) != nil then platforms.push "Mac OSX"; end
    if parsed.mac64(pdoc) != nil then platforms.push "Mac x64"; end
    if parsed.lin(pdoc) != nil then platforms.push "Linux"; end
    if parsed.ios(pdoc) != nil then platforms.push "iOS"; end
    if parsed.droid(pdoc) != nil then platforms.push "Android"; end
  end
  def function(ptext, capabilities)
    txt=Txt.new
    if txt.inst(ptext) != nil then capabilities.push "Instrument"; end
    if txt.eff(ptext) != nil then capabilities.push "Effect"; end
    if txt.chinst(ptext) != nil then capabilities.push "Instrument Host"; end
    if txt.cheff(ptext) != nil then capabilities.push "Effect Host"; end
  end
end
print 'Stage two.'
lnk='plinks.txt'
plinks=File.readlines(lnk)
x=0
y=plinks.length
y-=1 #zero to y
while x <= y
  if x.to_s.length == 1 then filename="json/000#{x}.json"
  elsif x.to_s.length == 2 then filename="json/00#{x}.json"
  elsif x.to_s.length == 3 then filename="json/0#{x}.json"
  else filename="json/#{x}.json"
  end
  if FileTest.exist?(filename) == true
    print 'x'
    x+=1
  else
    formats=[]
    capabilities=[]
    platforms=[]
    query=plinks[x].chomp
    kvr=MetaInspector.new(query) #this data needs alot of work
    pdoc=kvr.parsed
    ptext=pdoc.text
    if ptext =~ /product is no longer listed at KVR Audio/ || pdoc.at_css('#pluginsblock div p') == nil
      print 'x'
      x+=1
    else
      pachinko=Pachinko.new
      pachinko.type(pdoc, formats)
      pachinko.os(pdoc, platforms)
      pachinko.function(ptext, capabilities)
      hash=kvr.to_hash
{'fraction_non_alphanumeric': 0.1383388704318937, 'fraction_numerical': 0.015282392026578074, 'mean_word_length': 3.112568306010929, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2295002', 'n_tokens_mistral': 3019, 'n_tokens_neox': 2969, 'n_words': 746}