<commit_before>-- libquvi-scripts
-- Copyright (C) 2012-2013  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This program is free software: you can redistribute it and/or
-- modify it under the terms of the GNU Affero General Public
-- License as published by the Free Software Foundation, either
-- version 3 of the License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General
-- Public License along with this program.  If not, see
-- <http://www.gnu.org/licenses/>.
--

local TheOnion = {} -- Utility functions unique to this script.

-- Identify the media script.
function ident(qargs)
  return {
    can_parse_url = TheOnion.can_parse_url(qargs),
    domains = table.concat({'theonion.com'}, ',')
  }
end

-- Parse media URL.
function parse(self)
    self.host_id = "theonion"

    self.id = self.page_url:match(',(%d+)/') or error('no match: media ID')

    local u = string.format('http://theonion.com/videos/embed/%s.json',
                              self.id)

    local c = quvi.fetch(u, {fetch_type='config'})

    self.title = c:match('"title":%s+"(.-)"')
                  or error('no match: media title')

    self.thumbnail_url = c:match('"thumbnail": %["(.-)"%]') or ''

    self.url = {c:match('"video_url":%s+"(.-)"')
                  or error('no match: media stream URL')}

    return self
end

--
-- Utility functions
--

function TheOnion.can_parse_url(qargs)
  local U = require 'socket.url'
  local t = U.parse(qargs.input_url)
  if t and t.scheme and t.scheme:lower():match('^http$')
       and t.host   and t.host:lower():match('theonion%.com$')
       and t.path   and t.path:lower():match('^/video/.-,%d+/')
  then
    return true
  else
    return false
  end
end

-- vim: set ts=4 sw=4 tw=72 expandtab:
<commit_msg>media/theonion.lua: Rewrite parse function<commit_after>-- libquvi-scripts
-- Copyright (C) 2012-2013  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This program is free software: you can redistribute it and/or
-- modify it under the terms of the GNU Affero General Public
-- License as published by the Free Software Foundation, either
-- version 3 of the License, or (at your option) any later version.
--
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-- GNU Affero General Public License for more details.
--
-- You should have received a copy of the GNU Affero General
-- Public License along with this program.  If not, see
-- <http://www.gnu.org/licenses/>.
--

local TheOnion = {} -- Utility functions unique to this script.

-- Identify the media script.
function ident(qargs)
  return {
    can_parse_url = TheOnion.can_parse_url(qargs),
    domains = table.concat({'theonion.com'}, ',')
  }
end

-- Parse the media properties.
function parse(qargs)

  -- Make mandatory: the ID is required for the json URL.
  qargs.id = qargs.input_url:match(',(%d+)/') or error('no match: media ID')

  local u = string.format('http://theonion.com/videos/embed/%s.json',qargs.id)
  local d = quvi.http.fetch(u).data
  local J = require 'json'
  local j = J.decode(d)

  qargs.thumb_url = j['thumbnail'][1] or ''

  qargs.title = j['title'] or ''

  qargs.streams = TheOnion.iter_streams(j)

  return qargs
end

--
-- Utility functions
--

function TheOnion.can_parse_url(qargs)
  local U = require 'socket.url'
  local t = U.parse(qargs.input_url)
  if t and t.scheme and t.scheme:lower():match('^http$')
       and t.host   and t.host:lower():match('theonion%.com$')
       and t.path   and t.path:lower():match('^/video/.-,%d+/')
  then
    return true
  else
    return false
  end
end

function TheOnion.iter_streams(j)
  local u = j['video_url'] or error('no match: media stream URL')
  local S = require 'quvi/stream'
  return {S.stream_new(u)}
end

-- vim: set ts=2 sw=2 tw=72 expandtab:

{'fraction_non_alphanumeric': 0.1392699372239014, 'fraction_numerical': 0.006277609858172518, 'mean_word_length': 3.1010486177311725, 'pattern_counts': {'":': 3, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7625884', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1375, 'n_words': 461}