<commit_before>/*****************************************************************************
 * http.c: HTTP input module
 *****************************************************************************
 * Copyright (C) 2001-2008 the VideoLAN team
 * $Id$
 *
 * Authors: <NAME> <<EMAIL>>
 *          <NAME> <<EMAIL>>
 *          <NAME> <rem # videolan.org>
 *          <NAME> <dionoea at videolan dot org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <vlc_common.h>
#include <vlc_plugin.h>


#include <vlc_access.h>

#include <vlc_dialog.h>
#include <vlc_meta.h>
#include <vlc_network.h>
#include <vlc_url.h>
#include <vlc_tls.h>
#include <vlc_strings.h>
#include <vlc_charset.h>
#include <vlc_input.h>
#include <vlc_md5.h>
#include <vlc_http.h>

#ifdef HAVE_ZLIB_H
#   include <zlib.h>
#endif

#include <assert.h>

#ifdef HAVE_LIBPROXY
#    include <proxy.h>
#endif

#ifdef WIN32
#   include <windows.h>
#endif

/*****************************************************************************
 * Module descriptor
 *****************************************************************************/
static int  Open ( vlc_object_t * );
static void Close( vlc_object_t * );

#define PROXY_TEXT N_("HTTP proxy")
#define PROXY_LONGTEXT N_( \
    "HTTP proxy to be used It must be of the form " \
    "http://[user@]myproxy.mydomain:myport/ ; " \
    "if empty, the http_proxy environment variable will be tried." )

#define PROXY_PASS_TEXT N_("HTTP proxy password")
#define PROXY_PASS_LONGTEXT N_( \
    "If your HTTP proxy requires a password, set it here." )

#define CACHING_TEXT N_("Caching value in ms")
#define CACHING_LONGTEXT N_( \
    "Caching value for HTTP streams. This " \
    "value should be set in milliseconds." )

#define RECONNECT_TEXT N_("Auto re-connect")
#define RECONNECT_LONGTEXT N_( \
    "Automatically try to reconnect to the stream in case of a sudden " \
    "disconnect." )

#define CONTINUOUS_TEXT N_("Continuous stream")
#define CONTINUOUS_LONGTEXT N_("Read a file that is " \
    "being constantly updated (for example, a JPG file on a server). " \
    "You should not globally enable this option as it will break all other " \
    "types of HTTP streams." )

#define FORWARD_COOKIES_TEXT N_("Forward Cookies")
#define FORWARD_COOKIES_LONGTEXT N_("Forward Cookies across http redirections.")

#define USE_IE_PROXY_TEXT N_("Use Internet Explorer entered HTTP proxy server")
#define USE_IE_PROXY_LONGTEXT N_("Use Internet Explorer entered HTTP proxy " \
    "server for all URL. Don't take into account bypasses settings and auto " \
    "configuration scripts.")

#define REFERER_TEXT N_("HTTP referer value")
#define REFERER_LONGTEXT N_("Customize the HTTP referer, simulating a previous document")

#define UA_TEXT N_("User Agent")
#define UA_LONGTEXT N_("You can use a custom User agent or use a known one")

vlc_module_begin ()
    set_description( N_("HTTP input") )
    set_capability( "access", 0 )
    set_shortname( N_( "HTTP(S)" ) )
    set_category( CAT_INPUT )
    set_subcategory( SUBCAT_INPUT_ACCESS )

    add_string( "http-proxy", NULL, PROXY_TEXT, PROXY_LONGTEXT,
                false )
    add_password( "<PASSWORD>", NULL,
                  PROXY_PASS_TEXT, PROXY_PASS_LONGTEXT, false )
    add_integer( "http-caching", 4 * DEFAULT_PTS_DELAY / 1000,
                 CACHING_TEXT, CACHING_LONGTEXT, true )
        change_safe()
    add_string( "http-referrer", NULL, REFERER_TEXT, REFERER_LONGTEXT, false )
        change_safe()
    add_string( "http-user-agent", NULL, UA_TEXT, UA_LONGTEXT, false )
        change_safe()
        change_private()
    add_bool( "http-reconnect", false, RECONNECT_TEXT,
              RECONNECT_LONGTEXT, true )
    add_bool( "http-continuous", false, CONTINUOUS_TEXT,
              CONTINUOUS_LONGTEXT, true )
        change_safe()
    add_bool( "http-forward-cookies", true, FORWARD_COOKIES_TEXT,
              FORWARD_COOKIES_LONGTEXT, true )
#ifdef WIN32
    add_bool( "http-use-IE-proxy", false, USE_IE_PROXY_TEXT,
              USE_IE_PROXY_LONGTEXT, true )
#endif
    /* 'itpc' = iTunes Podcast */
    add_shortcut( "http", "https", "unsv", "itpc", "icyx" )
    set_callbacks( Open, Close )
vlc_module_end ()

/*****************************************************************************
 * Local prototypes
 *****************************************************************************/

struct access_sys_t
{
    int fd;
    bool b_error;
    tls_session_t *p_tls;
    v_socket_t    *p_vs;

    /* From uri */
    vlc_url_t url;
    char    *psz_user_agent;
    char    *psz_referrer;
    http_auth_t auth;

    /* Proxy */
    bool b_proxy;
    vlc_url_t  proxy;
    http_auth_t proxy_auth;
    char       *psz_proxy_passbuf;

    /* */
    int        i_code;
    const char *psz_protocol;
    int        i_version;

    char       *psz_mime;
    char       *psz_pragma;
    char       *psz_location;
    bool b_mms;
    bool b_icecast;
    bool b_ssl;
#ifdef HAVE_ZLIB_H
    bool b_compressed;
    struct
    {
        z_stream   stream;
        uint8_t   *p_buffer;
    } inflate;
#endif

    bool b_chunked;
    int64_t    i_chunk;

    int        i_icy_meta;
    uint64_t   i_icy_offset;
    char       *psz_icy_name;
    char       *psz_icy_genre;
    char       *psz_icy_title;

    uint64_t i_remaining;

    bool b_seekable;
    bool b_reconnect;
    bool b_continuous;
    bool b_pace_control;
    bool b_persist;
    bool b_has_size;

    vlc_array_t * cookies;
};

/* */
static int OpenWithCookies( vlc_object_t *p_this, const char *psz_access,
                            unsigned i_redirect, vlc_array_t *cookies );

/* */
static ssize_t Read( access_t *, uint8_t *, size_t );
static ssize_t ReadCompressed( access_t *, uint8_t *, size_t );
static int Seek( access_t *, uint64_t );
static int Control( access_t *, int, va_list );

/* */
static int Connect( access_t *, uint64_t );
static int Request( access_t *p_access, uint64_t i_tell );
static void Disconnect( access_t * );

/* Small Cookie utilities. Cookies support is partial. */
static char * cookie_get_content( const char * cookie );
static char * cookie_get_domain( const char * cookie );
static char * cookie_get_name( const char * cookie );
static void cookie_append( vlc_array_t * cookies, char * cookie );


static void AuthReply( access_t *p_acces, const char *psz_prefix,
                       vlc_url_t *p_url, http_auth_t *p_auth );
static int AuthCheckReply( access_t *p_access, const char *psz_header,
                           vlc_url_t *p_url, http_auth_t *p_auth );

/*****************************************************************************
 * Open:
 *****************************************************************************/
static int Open( vlc_object_t *p_this )
{
    access_t *p_access = (access_t*)p_this;
    return OpenWithCookies( p_this, p_access->psz_access, 5, NULL );
}

/**
 * Open the given url using the given cookies
 * @param p_this: the vlc object
 * @psz_access: the acces to use (http, https, ...) (this value must be used
 *              instead of p_access->psz_access)
 * @i_redirect: number of redirections remaining
 * @cookies: the available cookies
 * @return vlc error codes
 */
static int OpenWithCookies( vlc_object_t *p_this, const char *psz_access,
                            unsigned i_redirect, vlc_array_t *cookies )
{
    access_t     *p_access = (access_t*)p_this;
    access_sys_t *p_sys;
    char         *psz, *p;

    /* Only forward an store cookies if the corresponding option is activated */
    bool   b_forward_cookies = var_InheritBool( p_access, "http-forward-cookies" );
    vlc_array_t * saved_cookies = b_forward_cookies ? (cookies ? cookies : vlc_array_new()) : NULL;

    /* Set up p_access */
    STANDARD_READ_ACCESS_INIT;
#ifdef HAVE_ZLIB_H
    p_access->pf_read = ReadCompressed;
#endif
    p_sys->fd = -1;
    p_sys->b_proxy = false;
    p_sys->psz_proxy_passbuf = NULL;
    p_sys->i_version = 1;
    p_sys->b_seekable = true;
    p_sys->psz_mime = NULL;
    p_sys->psz_pragma = NULL;
    p_sys->b_mms = false;
    p_sys->b_icecast = false;
    p_sys->psz_location = NULL;
    p_sys->psz_user_agent = NULL;
    p_sys->psz_referrer = NULL;
    p_sys->b_pace_control = true;
    p_sys->b_ssl = false;
#ifdef HAVE_ZLIB_H
    p_sys->b_compressed = false;
    /* 15 is the max windowBits, +32 to enable optional gzip decoding */
    if( inflateInit2( &p_sys->inflate.stream, 32+15 ) != Z_OK )
        msg_Warn( p_access, "Error during zlib initialisation: %s",
                  p_sys->inflate.stream.msg );
    if( zlibCompileFlags() & (1<<17) )
        msg_Warn( p_access, "Your zlib was compiled without gzip support." );
    p_sys->inflate.p_buffer = NULL;
#endif
    p_sys->p_tls = NULL;
    p_sys->p_vs = NULL;
    p_sys->i_icy_meta = 0;
    p_sys->i_icy_offset = 0;
    p_sys->psz_icy_name = NULL;
    p_sys->psz_icy_genre = NULL;
    p_sys->psz_icy_title = NULL;
    p_sys->i_remaining = 0;
    p_sys->b_persist = false;
    p_sys->b_has_size = false;
    p_access->info.i_size = 0;
    p_access->info.i_pos  = 0;
    p_access->info.b_eof  = false;

    p_sys->cookies = saved_cookies;

    http_auth_Init( &p_sys->auth );
    http_auth_Init( &p_sys->proxy_auth );

    /* Parse URI - remove spaces */
    p = psz = strdup( p_access->psz_location );
    while( (p = strchr( p, ' ' )) != NULL )
        *p = '+';
    vlc_UrlParse( &p_sys->url, psz, 0 );
    free( psz );

    if( p_sys->url.psz_host == NULL || *p_sys->url.psz_host == '\0' )
    {
        msg_Warn( p_access, "invalid host" );
        goto error;
    }
    if( !strncmp( psz_access, "https", 5 ) )
    {
        /* HTTP over SSL */
        p_sys->b_ssl = true;
        if( p_sys->url.i_port <= 0 )
            p_sys->url.i_port = 443;
    }
    else
    {
        if( p_sys->url.i_port <= 0 )
            p_sys->url.i_port = 80;
    }

    /* Determine the HTTP user agent */
    /* See RFC2616 §2.2 token definition and §3.8 user-agent header */
    p_sys->psz_user_agent = var_InheritString( p_access, "http-user-agent" );
    if (p_sys->psz_user_agent)
    {
        for( char *p = p_sys->psz_user_agent; *p; p++ )
        {
            uint8_t c = *p;
            if( c < 32 || strchr( "()<>@,;:\\\"[]?={}", c ) )
                *p = '_'; /* remove potentially harmful characters */
        }
    }

    /* HTTP referrer */
    p_sys->psz_referrer = var_InheritString( p_access, "http-referrer" );

    /* Check proxy */
    psz = var_InheritString( p_access, "http-proxy" );
    if( psz )
    {
        p_sys->b_proxy = true;
        vlc_UrlParse( &p_sys->proxy, psz, 0 );
        free( psz );
    }
#ifdef HAVE_LIBPROXY
    else
    {
        pxProxyFactory *pf = px_proxy_factory_new();
        if (pf)
        {
            char *buf;
            int i;
            i=asprintf(&buf, "%s://%s", psz_access, p_access->psz_location);
            if (i >= 0)
            {
                msg_Dbg(p_access, "asking libproxy about url '%s'", buf);
                char **proxies = px_proxy_factory_get_proxies(pf, buf);
                if (proxies[0])
                {
                    msg_Dbg(p_access, "libproxy suggest to use '%s'", proxies[0]);
                    if(strcmp(proxies[0],"direct://") != 0)
                    {
                        p_sys->b_proxy = true;
                        vlc_UrlParse( &p_sys->proxy, proxies[0], 0);
                    }
                }
                for(i=0;proxies[i];i++) free(proxies[i]);
                free(proxies);
                free(buf);
            }
            px_proxy_factory_free(pf);
        }
        else
        {
            msg_Err(p_access, "Allocating memory for libproxy failed");
        }
    }
#elif defined( WIN32 )
    else
    {
        if( var_InheritBool( p_access, "http-use-IE-proxy" ) )
        {
            /* Try to get the proxy server address from Windows internet settings using registry. */
            HKEY h_key;
            /* Open the key */
            if( RegOpenKeyEx( HKEY_CURRENT_USER, "Software\\Microsoft" \
                              "\\Windows\\CurrentVersion\\Internet Settings",
                              0, KEY_READ, &h_key ) == ERROR_SUCCESS )
            {
                DWORD i_dataReadSize = 4; /* sizeof( DWORD ); */
                BYTE proxyEnable = 0;
                /* Get the proxy enable value */
                if( RegQueryValueEx( h_key, "ProxyEnable", NULL, NULL,
                                     &proxyEnable, &i_dataReadSize )
                                     == ERROR_SUCCESS )
                {
                    if( proxyEnable )
                    {
                        /* Proxy is enable */
                        char psz_key[256];
                        i_dataReadSize = 256;
                        if( RegQueryValueEx( h_key, "ProxyServer",
                                             NULL, NULL, (unsigned char *)psz_key,
                                             &i_dataReadSize )
                                             == ERROR_SUCCESS )
                        {
                            /* Get the proxy URL :
                            Proxy server value in the registry can be something like "address:port"
                            or "ftp=address1:port1;http=address2:port2 ..." depending of the
                            confirguration. */
                            char *psz_proxy;
                            psz_proxy = strstr( psz_key, "http=" );
                            if( psz_proxy != NULL )
                            {
                                psz_proxy += strlen( "http=" );
                                char *psz_endUrl = strchr( psz_proxy, ';' );
                                if( psz_endUrl != NULL )
                                    *psz_endUrl = '\0';
                            }
                            else
                                psz_proxy = psz_key;
                            /* Set proxy enable for this connection. */
                            p_sys->b_proxy = true;
                            vlc_UrlParse( &p_sys->proxy, psz_proxy, 0 );
                        }
                        msg_Warn( p_access, "Couldn't read in registry " \
                                  "the proxy server address." );
                    }
                }
                else
                    msg_Warn( p_access, "Couldn't read in registry if the " \
                              "proxy is enable or not." );
            }
            else
                msg_Warn( p_access, "Couldn't open internet settings key " \
                          "in registry." );
        }
    }
#else
    else
    {
        psz = getenv( "http_proxy" );
        if( psz )
        {<commit_msg>HTTP: remove --http-use-IE-proxy and assume it is always true<commit_after>/*****************************************************************************
 * http.c: HTTP input module
 *****************************************************************************
 * Copyright (C) 2001-2008 the VideoLAN team
 * $Id$
 *
 * Authors: <NAME> <<EMAIL>>
 *          <NAME> <<EMAIL>>
 *          <NAME> <rem # videolan.org>
 *          <NAME> <dionoea at videolan dot org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
 *****************************************************************************/

/*****************************************************************************
 * Preamble
 *****************************************************************************/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <vlc_common.h>
#include <vlc_plugin.h>


#include <vlc_access.h>

#include <vlc_dialog.h>
#include <vlc_meta.h>
#include <vlc_network.h>
#include <vlc_url.h>
#include <vlc_tls.h>
#include <vlc_strings.h>
#include <vlc_charset.h>
#include <vlc_input.h>
#include <vlc_md5.h>
#include <vlc_http.h>

#ifdef HAVE_ZLIB_H
#   include <zlib.h>
#endif

#include <assert.h>

#ifdef HAVE_LIBPROXY
#    include <proxy.h>
#endif

#ifdef WIN32
#   include <windows.h>
#endif

/*****************************************************************************
 * Module descriptor
 *****************************************************************************/
static int  Open ( vlc_object_t * );
static void Close( vlc_object_t * );

#define PROXY_TEXT N_("HTTP proxy")
#define PROXY_LONGTEXT N_( \
    "HTTP proxy to be used It must be of the form " \
    "http://[user@]myproxy.mydomain:myport/ ; " \
    "if empty, the http_proxy environment variable will be tried." )

#define PROXY_PASS_TEXT N_("HTTP proxy password")
#define PROXY_PASS_LONGTEXT N_( \
    "If your HTTP proxy requires a password, set it here." )

#define CACHING_TEXT N_("Caching value in ms")
#define CACHING_LONGTEXT N_( \
    "Caching value for HTTP streams. This " \
    "value should be set in milliseconds." )

#define RECONNECT_TEXT N_("Auto re-connect")
#define RECONNECT_LONGTEXT N_( \
    "Automatically try to reconnect to the stream in case of a sudden " \
    "disconnect." )

#define CONTINUOUS_TEXT N_("Continuous stream")
#define CONTINUOUS_LONGTEXT N_("Read a file that is " \
    "being constantly updated (for example, a JPG file on a server). " \
    "You should not globally enable this option as it will break all other " \
    "types of HTTP streams." )

#define FORWARD_COOKIES_TEXT N_("Forward Cookies")
#define FORWARD_COOKIES_LONGTEXT N_("Forward Cookies across http redirections.")

#define REFERER_TEXT N_("HTTP referer value")
#define REFERER_LONGTEXT N_("Customize the HTTP referer, simulating a previous document")

#define UA_TEXT N_("User Agent")
#define UA_LONGTEXT N_("You can use a custom User agent or use a known one")

vlc_module_begin ()
    set_description( N_("HTTP input") )
    set_capability( "access", 0 )
    set_shortname( N_( "HTTP(S)" ) )
    set_category( CAT_INPUT )
    set_subcategory( SUBCAT_INPUT_ACCESS )

    add_string( "http-proxy", NULL, PROXY_TEXT, PROXY_LONGTEXT,
                false )
    add_password( "<PASSWORD>", NULL,
                  PROXY_PASS_TEXT, PROXY_PASS_LONGTEXT, false )
    add_integer( "http-caching", 4 * DEFAULT_PTS_DELAY / 1000,
                 CACHING_TEXT, CACHING_LONGTEXT, true )
        change_safe()
    add_string( "http-referrer", NULL, REFERER_TEXT, REFERER_LONGTEXT, false )
        change_safe()
    add_string( "http-user-agent", NULL, UA_TEXT, UA_LONGTEXT, false )
        change_safe()
        change_private()
    add_bool( "http-reconnect", false, RECONNECT_TEXT,
              RECONNECT_LONGTEXT, true )
    add_bool( "http-continuous", false, CONTINUOUS_TEXT,
              CONTINUOUS_LONGTEXT, true )
        change_safe()
    add_bool( "http-forward-cookies", true, FORWARD_COOKIES_TEXT,
              FORWARD_COOKIES_LONGTEXT, true )
    /* 'itpc' = iTunes Podcast */
    add_shortcut( "http", "https", "unsv", "itpc", "icyx" )
    set_callbacks( Open, Close )
vlc_module_end ()

/*****************************************************************************
 * Local prototypes
 *****************************************************************************/

struct access_sys_t
{
    int fd;
    bool b_error;
    tls_session_t *p_tls;
    v_socket_t    *p_vs;

    /* From uri */
    vlc_url_t url;
    char    *psz_user_agent;
    char    *psz_referrer;
    http_auth_t auth;

    /* Proxy */
    bool b_proxy;
    vlc_url_t  proxy;
    http_auth_t proxy_auth;
    char       *psz_proxy_passbuf;

    /* */
    int        i_code;
    const char *psz_protocol;
    int        i_version;

    char       *psz_mime;
    char       *psz_pragma;
    char       *psz_location;
    bool b_mms;
    bool b_icecast;
    bool b_ssl;
#ifdef HAVE_ZLIB_H
    bool b_compressed;
    struct
    {
        z_stream   stream;
        uint8_t   *p_buffer;
    } inflate;
#endif

    bool b_chunked;
    int64_t    i_chunk;

    int        i_icy_meta;
    uint64_t   i_icy_offset;
    char       *psz_icy_name;
    char       *psz_icy_genre;
    char       *psz_icy_title;

    uint64_t i_remaining;

    bool b_seekable;
    bool b_reconnect;
    bool b_continuous;
    bool b_pace_control;
    bool b_persist;
    bool b_has_size;

    vlc_array_t * cookies;
};

/* */
static int OpenWithCookies( vlc_object_t *p_this, const char *psz_access,
                            unsigned i_redirect, vlc_array_t *cookies );

/* */
static ssize_t Read( access_t *, uint8_t *, size_t );
static ssize_t ReadCompressed( access_t *, uint8_t *, size_t );
static int Seek( access_t *, uint64_t );
static int Control( access_t *, int, va_list );

/* */
static int Connect( access_t *, uint64_t );
static int Request( access_t *p_access, uint64_t i_tell );
static void Disconnect( access_t * );

/* Small Cookie utilities. Cookies support is partial. */
static char * cookie_get_content( const char * cookie );
static char * cookie_get_domain( const char * cookie );
static char * cookie_get_name( const char * cookie );
static void cookie_append( vlc_array_t * cookies, char * cookie );


static void AuthReply( access_t *p_acces, const char *psz_prefix,
                       vlc_url_t *p_url, http_auth_t *p_auth );
static int AuthCheckReply( access_t *p_access, const char *psz_header,
                           vlc_url_t *p_url, http_auth_t *p_auth );

/*****************************************************************************
 * Open:
 *****************************************************************************/
static int Open( vlc_object_t *p_this )
{
    access_t *p_access = (access_t*)p_this;
    return OpenWithCookies( p_this, p_access->psz_access, 5, NULL );
}

/**
 * Open the given url using the given cookies
 * @param p_this: the vlc object
 * @psz_access: the acces to use (http, https, ...) (this value must be used
 *              instead of p_access->psz_access)
 * @i_redirect: number of redirections remaining
 * @cookies: the available cookies
 * @return vlc error codes
 */
static int OpenWithCookies( vlc_object_t *p_this, const char *psz_access,
                            unsigned i_redirect, vlc_array_t *cookies )
{
    access_t     *p_access = (access_t*)p_this;
    access_sys_t *p_sys;
    char         *psz, *p;

    /* Only forward an store cookies if the corresponding option is activated */
    bool   b_forward_cookies = var_InheritBool( p_access, "http-forward-cookies" );
    vlc_array_t * saved_cookies = b_forward_cookies ? (cookies ? cookies : vlc_array_new()) : NULL;

    /* Set up p_access */
    STANDARD_READ_ACCESS_INIT;
#ifdef HAVE_ZLIB_H
    p_access->pf_read = ReadCompressed;
#endif
    p_sys->fd = -1;
    p_sys->b_proxy = false;
    p_sys->psz_proxy_passbuf = NULL;
    p_sys->i_version = 1;
    p_sys->b_seekable = true;
    p_sys->psz_mime = NULL;
    p_sys->psz_pragma = NULL;
    p_sys->b_mms = false;
    p_sys->b_icecast = false;
    p_sys->psz_location = NULL;
    p_sys->psz_user_agent = NULL;
    p_sys->psz_referrer = NULL;
    p_sys->b_pace_control = true;
    p_sys->b_ssl = false;
#ifdef HAVE_ZLIB_H
    p_sys->b_compressed = false;
    /* 15 is the max windowBits, +32 to enable optional gzip decoding */
    if( inflateInit2( &p_sys->inflate.stream, 32+15 ) != Z_OK )
        msg_Warn( p_access, "Error during zlib initialisation: %s",
                  p_sys->inflate.stream.msg );
    if( zlibCompileFlags() & (1<<17) )
        msg_Warn( p_access, "Your zlib was compiled without gzip support." );
    p_sys->inflate.p_buffer = NULL;
#endif
    p_sys->p_tls = NULL;
    p_sys->p_vs = NULL;
    p_sys->i_icy_meta = 0;
    p_sys->i_icy_offset = 0;
    p_sys->psz_icy_name = NULL;
    p_sys->psz_icy_genre = NULL;
    p_sys->psz_icy_title = NULL;
    p_sys->i_remaining = 0;
    p_sys->b_persist = false;
    p_sys->b_has_size = false;
    p_access->info.i_size = 0;
    p_access->info.i_pos  = 0;
    p_access->info.b_eof  = false;

    p_sys->cookies = saved_cookies;

    http_auth_Init( &p_sys->auth );
    http_auth_Init( &p_sys->proxy_auth );

    /* Parse URI - remove spaces */
    p = psz = strdup( p_access->psz_location );
    while( (p = strchr( p, ' ' )) != NULL )
        *p = '+';
    vlc_UrlParse( &p_sys->url, psz, 0 );
    free( psz );

    if( p_sys->url.psz_host == NULL || *p_sys->url.psz_host == '\0' )
    {
        msg_Warn( p_access, "invalid host" );
        goto error;
    }
    if( !strncmp( psz_access, "https", 5 ) )
    {
        /* HTTP over SSL */
        p_sys->b_ssl = true;
        if( p_sys->url.i_port <= 0 )
            p_sys->url.i_port = 443;
    }
    else
    {
        if( p_sys->url.i_port <= 0 )
            p_sys->url.i_port = 80;
    }

    /* Determine the HTTP user agent */
    /* See RFC2616 §2.2 token definition and §3.8 user-agent header */
    p_sys->psz_user_agent = var_InheritString( p_access, "http-user-agent" );
    if (p_sys->psz_user_agent)
    {
        for( char *p = p_sys->psz_user_agent; *p; p++ )
        {
            uint8_t c = *p;
            if( c < 32 || strchr( "()<>@,;:\\\"[]?={}", c ) )
                *p = '_'; /* remove potentially harmful characters */
        }
    }

    /* HTTP referrer */
    p_sys->psz_referrer = var_InheritString( p_access, "http-referrer" );

    /* Check proxy */
    psz = var_InheritString( p_access, "http-proxy" );
    if( psz )
    {
        p_sys->b_proxy = true;
        vlc_UrlParse( &p_sys->proxy, psz, 0 );
        free( psz );
    }
#ifdef HAVE_LIBPROXY
    else
    {
        pxProxyFactory *pf = px_proxy_factory_new();
        if (pf)
        {
            char *buf;
            int i;
            i=asprintf(&buf, "%s://%s", psz_access, p_access->psz_location);
            if (i >= 0)
            {
                msg_Dbg(p_access, "asking libproxy about url '%s'", buf);
                char **proxies = px_proxy_factory_get_proxies(pf, buf);
                if (proxies[0])
                {
                    msg_Dbg(p_access, "libproxy suggest to use '%s'", proxies[0]);
                    if(strcmp(proxies[0],"direct://") != 0)
                    {
                        p_sys->b_proxy = true;
                        vlc_UrlParse( &p_sys->proxy, proxies[0], 0);
                    }
                }
                for(i=0;proxies[i];i++) free(proxies[i]);
                free(proxies);
                free(buf);
            }
            px_proxy_factory_free(pf);
        }
        else
        {
            msg_Err(p_access, "Allocating memory for libproxy failed");
        }
    }
#elif defined( WIN32 )
    else
    {
        /* Try to get the proxy server address from Windows internet settings using registry. */
        HKEY h_key;
        /* Open the key */
        if( RegOpenKeyEx( HKEY_CURRENT_USER, "Software\\Microsoft"
                          "\\Windows\\CurrentVersion\\Internet Settings",
                          0, KEY_READ, &h_key ) == ERROR_SUCCESS )
        {
            DWORD len = sizeof( DWORD );
            BYTE proxyEnable;

            /* Get the proxy enable value */
            if( RegQueryValueEx( h_key, "ProxyEnable", NULL, NULL,
                                 &proxyEnable, &len ) == ERROR_SUCCESS
             && proxyEnable )
            {
                /* Proxy is enabled */
                /* Get the proxy URL :
                   Proxy server value in the registry can be something like "address:port"
                   or "ftp=address1:port1;http=address2:port2 ..." depending of the
                   confirguration. */
                unsigned char key[256];

                len = sizeof( key );
                if( RegQueryValueEx( h_key, "ProxyServer", NULL, NULL,
                                     key, &len ) == ERROR_SUCCESS )
                {
                    /* FIXME: This is lame. The string should be tokenized. */
#warning FIXME.
                    char *psz_proxy = strstr( (char *)key, "http=" );
                    if( psz_proxy != NULL )
                    {
                        psz_proxy += 5;
                        char *end = strchr( psz_proxy, ';' );
                        if( end != NULL )
                            *end = '\0';
                    }
                    else
                        psz_proxy = (char *)key;
                    /* Set proxy enable for this connection. */
                    p_sys->b_proxy = true;
                    vlc_UrlParse( &p_sys->proxy, psz_proxy, 0 );
                }
            }
            else
                msg_Dbg( p_access, "HTTP proxy disabled (MSIE)" );
        }
    }
#else
    else
    {
        psz = getenv( "http_proxy" );
        if( psz )
        {
{'fraction_non_alphanumeric': 0.15798708823042543, 'fraction_numerical': 0.007283562324118523, 'mean_word_length': 2.249354561101549, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 177, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733375', 'n_tokens_mistral': 9791, 'n_tokens_neox': 9534, 'n_words': 2439}