<commit_before>
-- libquvi-scripts
-- Copyright (C) 2010-2011  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This library is free software; you can redistribute it and/or
-- modify it under the terms of the GNU Lesser General Public
-- License as published by the Free Software Foundation; either
-- version 2.1 of the License, or (at your option) any later version.
--
-- This library is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- Lesser General Public License for more details.
--
-- You should have received a copy of the GNU Lesser General Public
-- License along with this library; if not, write to the Free Software
-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-- 02110-1301  USA
--

-- "http://dai.ly/cityofscars",
-- "http://www.dailymotion.com/video/xdpig1_city-of-scars_shortfilms",

local Dailymotion = {} -- Utility functions unique to this script.

-- Identify the script.
function ident(self)
    package.path = self.script_dir .. '/?.lua'
    local C      = require 'quvi/const'
    local r      = {}
    r.domain     = "dailymotion%.%w+"
    r.formats    = "default|best"
    r.categories = C.proto_http
    local U      = require 'quvi/util'
    r.handles    = U.handles(self.page_url, {r.domain, "dai.ly"},
                    {"/video/","/%w+$","/family_filter"})
    return r
end

-- Query available formats.
function query_formats(self)
    local U       = require 'quvi/util'
    local page    = Dailymotion.fetch_page(self, U)
    local formats = Dailymotion.iter_formats(page, U)

    local t = {}
    for _,v in pairs(formats) do
        table.insert(t, Dailymotion.to_s(v))
    end

    table.sort(t)
    self.formats = table.concat(t, "|")

    return self
end

-- Parse media URL.
function parse(self)
    self.host_id = "dailymotion"

    local U    = require 'quvi/util'
    local page = Dailymotion.fetch_page(self, U)

    local _,_,s = page:find('title="(.-)"')
    self.title  = s or error("no match: media title")

    local _,_,s = page:find("video/(.-)_")
    self.id     = s or error("no match: media id")

    local _,_,s = page:find('"og:image" content="(.-)"')
    self.thumbnail_url = s or ''

    local formats = Dailymotion.iter_formats(page, U)
    local format  = U.choose_format(self, formats,
                                     Dailymotion.choose_best,
                                     Dailymotion.choose_default,
                                     Dailymotion.to_s)
                        or error("unable to choose format")
    self.url      = {format.url or error("no match: media url")}
    return self
end

--
-- Utility functions
--

function Dailymotion.fetch_page(self, U)
    self.page_url = Dailymotion.normalize(self.page_url)

    local _,_,s = self.page_url:find('/family_filter%?urlback=(.+)')
    if s then
        self.page_url = 'http://dailymotion.com' .. U.unescape(s)
    end

    local opts = {arbitrary_cookie = 'family_filter=off'}
    return quvi.fetch(self.page_url, opts)
end

function Dailymotion.normalize(page_url) -- "Normalize" embedded URLs
    if page_url:find("/swf/") then
        page_url = page_url:gsub("/swf/", "/")
    elseif page_url:find("/embed/") then
        page_url = page_url:gsub("/embed/", "/")
    end
    return page_url
end

function Dailymotion.iter_formats(page, U)
    local _,_,seq = page:find('"sequence",%s+"(.-)"')
    if not seq then
        local e = "no match: sequence"
        if page:find("_partnerplayer") then
            e = e .. ": looks like a partner video which we do not support"
        end
        error(e)
    end
    seq = U.unescape(seq)
--[[
    local _,_,msg = seq:find('"message":"(.-)[<"]')
    if msg then
        msg = msg:gsub('+',' ')
        error(msg:gsub('\\',''))
    end
]]--
    local _,_,vpp = seq:find('"videoPluginParameters":{(.-)}')
    if not vpp then
        -- See also <http://sourceforge.net/apps/trac/clive/ticket/4>
        error("no match: video plugin params")
    end

    local t = {}
    for url in vpp:gfind('%w+URL":"(.-)"') do
        url = url:gsub("\\/", "/")
        local _,_,c,w,h,cn = url:find('(%w+)%-(%d+)x(%d+).-%.(%w+)')
        if not c then
            error('no match: codec, width, height, container')
        end
--        print(c,w,h,cn)
        table.insert(t, {width=tonumber(w), height=tonumber(h),
                         container=cn,      codec=string.lower(c),
                         url=url})
    end

    return t
end

function Dailymotion.choose_default(formats) -- Lowest quality available
    local r = {width=0xffff, height=0xffff, url=nil}
    local U = require 'quvi/util'
    for _,v in pairs(formats) do
        if U.is_lower_quality(v,r) then
            r = v
        end
    end
--    for k,v in pairs(r) do print(k,v) end<commit_msg>FIX: dailymotion.lua:130: no match: video plugin params<commit_after>
-- libquvi-scripts
-- Copyright (C) 2010-2011  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This library is free software; you can redistribute it and/or
-- modify it under the terms of the GNU Lesser General Public
-- License as published by the Free Software Foundation; either
-- version 2.1 of the License, or (at your option) any later version.
--
-- This library is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- Lesser General Public License for more details.
--
-- You should have received a copy of the GNU Lesser General Public
-- License along with this library; if not, write to the Free Software
-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-- 02110-1301  USA
--

-- "http://dai.ly/cityofscars",
-- "http://www.dailymotion.com/video/xdpig1_city-of-scars_shortfilms",

local Dailymotion = {} -- Utility functions unique to this script.

-- Identify the script.
function ident(self)
    package.path = self.script_dir .. '/?.lua'
    local C      = require 'quvi/const'
    local r      = {}
    r.domain     = "dailymotion%.%w+"
    r.formats    = "default|best"
    r.categories = C.proto_http
    local U      = require 'quvi/util'
    r.handles    = U.handles(self.page_url, {r.domain, "dai.ly"},
                    {"/video/","/%w+$","/family_filter"})
    return r
end

-- Query available formats.
function query_formats(self)
    local U       = require 'quvi/util'
    local page    = Dailymotion.fetch_page(self, U)
    local formats = Dailymotion.iter_formats(page, U)

    local t = {}
    for _,v in pairs(formats) do
        table.insert(t, Dailymotion.to_s(v))
    end

    table.sort(t)
    self.formats = table.concat(t, "|")

    return self
end

-- Parse media URL.
function parse(self)
    self.host_id = "dailymotion"

    local U    = require 'quvi/util'
    local page = Dailymotion.fetch_page(self, U)

    local _,_,s = page:find('title="(.-)"')
    self.title  = s or error("no match: media title")

    local _,_,s = page:find("video/(.-)_")
    self.id     = s or error("no match: media id")

    local _,_,s = page:find('"og:image" content="(.-)"')
    self.thumbnail_url = s or ''

    local formats = Dailymotion.iter_formats(page, U)
    local format  = U.choose_format(self, formats,
                                     Dailymotion.choose_best,
                                     Dailymotion.choose_default,
                                     Dailymotion.to_s)
                        or error("unable to choose format")
    self.url      = {format.url or error("no match: media url")}
    return self
end

--
-- Utility functions
--

function Dailymotion.fetch_page(self, U)
    self.page_url = Dailymotion.normalize(self.page_url)

    local _,_,s = self.page_url:find('/family_filter%?urlback=(.+)')
    if s then
        self.page_url = 'http://dailymotion.com' .. U.unescape(s)
    end

    local opts = {arbitrary_cookie = 'family_filter=off'}
    return quvi.fetch(self.page_url, opts)
end

function Dailymotion.normalize(page_url) -- "Normalize" embedded URLs
    if page_url:find("/swf/") then
        page_url = page_url:gsub("/swf/", "/")
    elseif page_url:find("/embed/") then
        page_url = page_url:gsub("/embed/", "/")
    end
    return page_url
end

function Dailymotion.iter_formats(page, U)
    local seq = page:match('"sequence",%s+"(.-)"')
    if not seq then
        local e = "no match: sequence"
        if page:find("_partnerplayer") then
            e = e .. ": looks like a partner video which we do not support"
        end
        error(e)
    end

    seq = U.unescape(seq)

    local t = {}
    for url in seq:gfind('%w+URL":"(.-)"') do
        local c,w,h,cn = url:match('(%w+)%-(%d+)x(%d+).-%.(%w+)')
        if c then
            table.insert(t, {width=tonumber(w), height=tonumber(h),
                             container=cn,      codec=string.lower(c),
                             url=url:gsub("\\/", "/")})
--            print(c,w,h,cn)
        end
    end

    if #t == 0 then
        error("no match: media URL")
    end

    return t
end

function Dailymotion.choose_default(formats) -- Lowest quality available
    local r = {width=0xffff, height=0xffff, url=nil}
    local U = require 'quvi/util'
    for _,v in pairs(formats) do
        if U.is_lower_quality(v,r) then
            r = v
        end
    end
--    for k,v in pairs(r) do print(k,v) end
{'fraction_non_alphanumeric': 0.1391929934313419, 'fraction_numerical': 0.00552601397143155, 'mean_word_length': 2.355019237495628, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7639043', 'n_tokens_mistral': 3226, 'n_tokens_neox': 3109, 'n_words': 869}