<commit_before>
-- libquvi-scripts
-- Copyright (C) 2010-2011  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This library is free software; you can redistribute it and/or
-- modify it under the terms of the GNU Lesser General Public
-- License as published by the Free Software Foundation; either
-- version 2.1 of the License, or (at your option) any later version.
--
-- This library is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- Lesser General Public License for more details.
--
-- You should have received a copy of the GNU Lesser General Public
-- License along with this library; if not, write to the Free Software
-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-- 02110-1301  USA
--

local YouTube = {} -- Utility functions unique to this script

-- <http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs>

-- Identify the script.
function ident(self)
    package.path = self.script_dir .. '/?.lua'
    local C      = require 'quvi/const'
    local r      = {}
    r.domain     = "youtube%.com"
    r.formats    = "default|best"
    r.categories  = C.proto_http
    self.page_url = YouTube.normalize(self.page_url)
    local U       = require 'quvi/util'
    r.handles     = U.handles(self.page_url,
                        {r.domain}, {"/watch"}, {"v=[%w-_]+"})
    return r
end

-- Query available formats.
function query_formats(self)
    local config,U  = YouTube.get_config(self)
    local formats   = YouTube.iter_formats(config, U)

    local t = {}
    for _,v in pairs(formats) do
        table.insert(t, YouTube.to_s(v))
    end

    table.sort(t)
    self.formats = table.concat(t, "|")

    return self
end

-- Parse URL.
function parse(self)
    self.host_id   = "youtube"
    local page_url = YouTube.normalize(self.page_url)

    local _,_,s = page_url:find('#a?t=(.+)')
    self.start_time = s or ''

    return YouTube.get_video_info(self)
end

--
-- Utility functions
--

function YouTube.normalize(s)
    if not s then return s end
    local U = require 'quvi/url'
    local t = U.parse(s)
    if not t.host then return s end
    t.host = t.host:gsub('youtu%.be', 'youtube.com')
    t.host = t.host:gsub('-nocookie', '')
    if t.path then
        local p = {'/embed/([-_%w]+)', '/%w/([-_%w]+)', '/([-_%w]+)'}
        for _,v in pairs(p) do
            local m = t.path:match(v)
            if m and #m == 11 then
                t.query = 'v=' .. m
                t.path  = '/watch'
            end
        end
    end
    return U.build(t)
end

function YouTube.get_config(self)
    local _,_,s  = self.page_url:find('^(%w+)://')
    local scheme = s or error("no match: scheme")

    local page_url = YouTube.normalize(self.page_url)

    local _,_,s = page_url:find("v=([%w-_]+)")
    self.id = s or error("no match: media id")

    local s_fmt = "%s://www.youtube.com/get_video_info?&video_id=%s"
                    .. "&el=detailpage&ps=default&eurl=&gl=US&hl=en"

    local config_url = string.format(s_fmt, scheme, self.id)

    local U      = require 'quvi/util'
    local config = U.decode(quvi.fetch(config_url, {fetch_type='config'}))

    if config['reason'] then
        local reason = U.unescape(config['reason'])
        local code   = config['errorcode']
        error(string.format("%s (code=%s)", reason, code))
    end

    return config,U
end

function YouTube.iter_formats(config, U)
    local fmt_stream_map = config['url_encoded_fmt_stream_map']
                        or error("no match: url_encoded_fmt_stream_map")

    fmt_stream_map = U.unescape(fmt_stream_map) .. ","

    local urls = {}
    for f in fmt_stream_map:gfind('([^,]*),') do
        local d = U.decode(f)
        if d['itag'] and d['url'] then
            urls[U.unescape(d['itag'])] = U.unescape(d['url'])
        end
    end

    local fmt_map = config['fmt_list'] or error("no match: fmt_list")
    fmt_map = U.unescape(fmt_map)
<commit_msg>youtube.lua: Use "foo:match" instead<commit_after>
-- libquvi-scripts
-- Copyright (C) 2010-2012  <NAME> <<EMAIL>>
--
-- This file is part of libquvi-scripts <http://quvi.sourceforge.net/>.
--
-- This library is free software; you can redistribute it and/or
-- modify it under the terms of the GNU Lesser General Public
-- License as published by the Free Software Foundation; either
-- version 2.1 of the License, or (at your option) any later version.
--
-- This library is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-- Lesser General Public License for more details.
--
-- You should have received a copy of the GNU Lesser General Public
-- License along with this library; if not, write to the Free Software
-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-- 02110-1301  USA
--

local YouTube = {} -- Utility functions unique to this script

-- <http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs>

-- Identify the script.
function ident(self)
    package.path = self.script_dir .. '/?.lua'
    local C      = require 'quvi/const'
    local r      = {}
    r.domain     = "youtube%.com"
    r.formats    = "default|best"
    r.categories  = C.proto_http
    self.page_url = YouTube.normalize(self.page_url)
    local U       = require 'quvi/util'
    r.handles     = U.handles(self.page_url,
                        {r.domain}, {"/watch"}, {"v=[%w-_]+"})
    return r
end

-- Query available formats.
function query_formats(self)
    local config,U  = YouTube.get_config(self)
    local formats   = YouTube.iter_formats(config, U)

    local t = {}
    for _,v in pairs(formats) do
        table.insert(t, YouTube.to_s(v))
    end

    table.sort(t)
    self.formats = table.concat(t, "|")

    return self
end

-- Parse URL.
function parse(self)
    self.host_id = "youtube"

    local p_url = YouTube.normalize(self.page_url)
    self.start_time = p_url:match('#a?t=(.+)') or ''

    return YouTube.get_video_info(self)
end

--
-- Utility functions
--

function YouTube.normalize(s)
    if not s then return s end
    local U = require 'quvi/url'
    local t = U.parse(s)
    if not t.host then return s end
    t.host = t.host:gsub('youtu%.be', 'youtube.com')
    t.host = t.host:gsub('-nocookie', '')
    if t.path then
        local p = {'/embed/([-_%w]+)', '/%w/([-_%w]+)', '/([-_%w]+)'}
        for _,v in pairs(p) do
            local m = t.path:match(v)
            if m and #m == 11 then
                t.query = 'v=' .. m
                t.path  = '/watch'
            end
        end
    end
    return U.build(t)
end

function YouTube.get_config(self)
    local sch = self.page_url:match('^(%w+)://')
                  or error("no match: scheme")

    local p_url = YouTube.normalize(self.page_url)

    self.id = p_url:match("v=([%w-_]+)")
                or error("no match: media ID")

    local s_fmt = "%s://www.youtube.com/get_video_info?&video_id=%s"
                    .. "&el=detailpage&ps=default&eurl=&gl=US&hl=en"

    local c_url = string.format(s_fmt, sch, self.id)

    local U = require 'quvi/util'
    local c = U.decode(quvi.fetch(c_url, {fetch_type='config'}))

    if c['reason'] then
        local reason = U.unescape(c['reason'])
        local code = c['errorcode']
        error(string.format("%s (code=%s)", reason, code))
    end

    return c, U
end

function YouTube.iter_formats(config, U)
    local fmt_stream_map = config['url_encoded_fmt_stream_map']
                        or error("no match: url_encoded_fmt_stream_map")

    fmt_stream_map = U.unescape(fmt_stream_map) .. ","

    local urls = {}
    for f in fmt_stream_map:gfind('([^,]*),') do
        local d = U.decode(f)
        if d['itag'] and d['url'] then
            urls[U.unescape(d['itag'])] = U.unescape(d['url'])
        end
    end

    local fmt_map = config['fmt_list'] or error("no match: fmt_list")
    fmt_map = U.unescape(fmt_map)

{'fraction_non_alphanumeric': 0.14243396692174773, 'fraction_numerical': 0.005677610466551469, 'mean_word_length': 2.5446194225721785, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '98108', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2670, 'n_words': 757}