<commit_before>-- libquvi-scripts
-- Copyright (C) 2013  <NAME> <<EMAIL>>
-- Copyright (C) 2011  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This program is free software: you can redistribute it and/or
-- modify it under the terms of the GNU Affero General Public
-- License as published by the Free Software Foundation, either
-- version 3 of the License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General
-- Public License along with this program.  If not, see
-- <http://www.gnu.org/licenses/>.
--

local Metacafe = {} -- Utility functions unique to this script.

-- Identify the media script.
function ident(qargs)
  return {
    can_parse_url = Metacafe.can_parse_url(qargs),
    domains = table.concat({'metacafe.com'}, ',')
  }
end

-- Parse media URL.
function parse(self)
    self.host_id = "metacafe"

    if Metacafe.redirectp(self) then
        return self
    end

    local U = require 'quvi/util'
    local p = Metacafe.fetch_page(self, U)

    local v = p:match('name="flashvars" value="(.-)"')
                or error('no match: flashvars')

    v = U.slash_unescape(U.unescape(v))

    self.thumbnail_url = p:match('rel="image_src" href="(.-)"') or ''

    self.title = v:match('title=(.-)&') or error('no match: media title')

    self.id = v:match('itemID=(%d+)') or error('no match: media ID')

    local u = v:match('"mediaURL":"(.-)"')
                or error('no match: media stream URL')

    local k = v:match('"key":"__gda__","value":"(.-)"')
                or error('no match: key')

    self.url = {string.format("%s?__gda__=%s", u, k)}

    return self
end

--
-- Utility functions
--

function Metacafe.can_parse_url(qargs)
  local U = require 'socket.url'
  local t = U.parse(qargs.input_url)
  if t and t.scheme and t.scheme:lower():match('^http$')
       and t.host   and t.host:lower():match('metacafe%.com$')
       and t.path   and (t.path:lower():match('^/watch/%d+/')
                          or t.path:lower():match('^/watch/yt-[^/]+/'))
  then
    return true
  else
    return false
  end
end

function Metacafe.redirectp(self)
    local s = self.page_url:match('/watch/yt%-([^/]+)/')
    if s then -- Hand over to youtube.lua
        self.redirect_url = 'http://youtube.com/watch?v=' .. s
        return true
    end
    return false
end

function Metacafe.fetch_page(self, U)
    self.page_url = Metacafe.normalize(self.page_url)

    return quvi.fetch(self.page_url)
end

function Metacafe.normalize(page_url) -- "Normalize" embedded URLs
    return page_url
end

-- vim: set ts=4 sw=4 tw=72 expandtab:<commit_msg>media/metacafe.lua: Rewrite parse function<commit_after>-- libquvi-scripts
-- Copyright (C) 2013  <NAME> <<EMAIL>>
-- Copyright (C) 2011  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This program is free software: you can redistribute it and/or
-- modify it under the terms of the GNU Affero General Public
-- License as published by the Free Software Foundation, either
-- version 3 of the License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General
-- Public License along with this program.  If not, see
-- <http://www.gnu.org/licenses/>.
--

local Metacafe = {} -- Utility functions unique to this script.

-- Identify the media script.
function ident(qargs)
  return {
    can_parse_url = Metacafe.can_parse_url(qargs),
    domains = table.concat({'metacafe.com'}, ',')
  }
end

-- Parse the media properties.
function parse(qargs)

  if Metacafe.is_affiliate(qargs) then
    return qargs
  end

  local p = quvi.http.fetch(qargs.input_url).data

  local v = p:match('name="flashvars" value="(.-)"')
              or error('no match: flashvars')

  qargs.thumb_url = p:match('"og:image"%s+content="(.-)"') or ''

  local U = require 'socket.url'
  local T = require 'quvi/util'

  local r = T.decode(U.unescape(v))

  qargs.streams = Metacafe.iter_streams(U, r)

  qargs.duration_ms = tonumber(r['duration'] or 0) * 1000

  qargs.title = r['title'] or ''

  qargs.id = r['itemID'] or ''

  return qargs
end

--
-- Utility functions
--

function Metacafe.can_parse_url(qargs)
  local U = require 'socket.url'
  local t = U.parse(qargs.input_url)
  if t and t.scheme and t.scheme:lower():match('^http$')
       and t.host   and t.host:lower():match('metacafe%.com$')
       and t.path   and (t.path:lower():match('^/watch/%d+/')
                          or t.path:lower():match('^/watch/yt-[^/]+/'))
  then
    return true
  else
    return false
  end
end

function Metacafe.is_affiliate(qargs)
  local id = qargs.input_url:match('/watch/yt%-([^/]+)/') -- YouTube.
  if id then
    qargs.goto_url = table.concat({'http://youtu.be/',id})
  end
  return qargs.goto_url ~= nil
end

function Metacafe.iter_streams(U, r)
  local S = require 'quvi/stream'
  local J = require 'json'

  local j = J.decode(r['mediaData'])
  local t = {}

  for k,_ in pairs(j) do
    local u = U.parse(j[k]['mediaURL'] or error('no match: media stream URL'))

    local q = {} -- Construct the URL query from the given args.
    for _,v in pairs(j[k]['access'][1]) do
      table.insert(q,v)
    end
    u.query = table.concat(q, '=')

    local s = S.stream_new(U.build(u)) -- Build stream URL from these elems.
    Metacafe.to_id(s,k)

    table.insert(t,s)
  end

  local r = {} -- Reverse the stream order, SD should be first ('default').
  for i = #t,1,-1 do
    table.insert(r, t[i])
  end

  if #r >1 then
    Metacafe.ch_best(S, r)
  end

  return r
end

function Metacafe.ch_best(S, t)
  t[#t].flags.best = true -- Make the last stream the best one.
end

function Metacafe.to_id(s,k)
  -- Lack of a better scheme: reuse the internal IDs with minor tweaks.
  s.id = k:gsub('(%l)(%u)','%1_%2'):lower()
end

-- vim: set ts=2 sw=2 tw=72 expandtab:
{'fraction_non_alphanumeric': 0.14826351035664226, 'fraction_numerical': 0.005762342314281264, 'mean_word_length': 2.993781094527363, 'pattern_counts': {'":': 3, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7625883', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2155, 'n_words': 664}