<commit_before>%%----------------------------------------------------------------------
%%% File    : yaws_server.erl
%%% Author  : <NAME> <<EMAIL>>
%%% Purpose : 
%%% Created : 16 Jan 2002 by <NAME> <<EMAIL>>
%%%----------------------------------------------------------------------

-module(yaws_server).
-author('<EMAIL>').

-compile(export_all).
%%-export([Function/Arity, ...]).

-behaviour(gen_server).

-include_lib("yaws/include/yaws.hrl").
-include_lib("yaws/include/yaws_api.hrl").
-include("yaws_debug.hrl").

-include_lib("kernel/include/file.hrl").

%% External exports
-export([start_link/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
-import(filename, [join/1]).
-import(lists, [foreach/2, map/2, flatten/1, flatmap/2, reverse/1]).

-import(yaws_api, [ehtml_expand/1]).

-record(gs, {gconf,
	     group,         %% list of #sconf{} s
	     ssl,           %% ssl | nossl
	     l,             %% listen socket
	     mnum = 0,      %% dyn compiled erl module  number
	     sessions = 0,  %% number of HTTP sessions
	     reqs = 0}).    %% number of HTTP requests


%%%----------------------------------------------------------------------
%%% API
%%%----------------------------------------------------------------------
start_link() ->
    gen_server:start_link({local, yaws_server}, yaws_server, [], []).

status() ->
    gen_server:call(?MODULE, status).



stats() -> 
    {S, Time} = status(),
    {_GC, Srvs, _} = S,
    Diff = calendar:time_difference(Time, calendar:local_time()),
    G = fun(L) -> lists:reverse(lists:keysort(2, L)) end,
    
    R= flatmap(
      fun({_Pid, SCS}) ->
	      map(
		fun(SC) ->
			
			E = SC#sconf.ets,
			L = ets:match(E, {{urlc_total, '$1'}, '$2'}),
			{SC#sconf.servername,  
			 flatten(yaws:fmt_ip(SC#sconf.listen)),
			 G(map(fun(P) -> list_to_tuple(P) end, L))}
		end, SCS)
      end, Srvs),
    {Diff, R}.

			      
get_app_args() ->
    AS=init:get_arguments(),
    Debug = case application:get_env(yaws, debug) of
		undefined ->
		    lists:member({yaws, ["debug"]}, AS);
		{ok, Val}  ->
		    Val
	    end,
    Trace = case application:get_env(yaws, trace) of
		undefined ->
		    case {lists:member({yaws, ["trace", "http"]}, AS),
			  lists:member({yaws, ["trace", "traffic"]}, AS)} of
			{true, _} ->
			    {true, http};
			{_, true} ->
			    {true, traffic};
			_ ->
			    false
		    end;
		{ok, http} ->
		    {true, http};
		{ok, traffic} ->
		    {true, traffic};
		_ ->
		    false
	    end,
    Conf = case application:get_env(yaws, conf) of
	       undefined ->
		   find_c(AS);
	       {ok, File} ->
		   {file, File}
	   end,
    RunMod = case application:get_env(yaws, runmod) of
		 undefined ->
		     find_runmod(AS);
		 {ok,Mod} ->
		     {ok,Mod}
	     end,
    Embed = case application:get_env(yaws, embedded) of
		undefined ->
		    false;
		{ok, Val0} ->
		    Val0
	    end,
    {Debug, Trace, Conf, RunMod, Embed}.

find_c([{conf, [File]} |_]) ->
    {file, File};
find_c([_|T]) ->
    find_c(T);
find_c([]) ->
    false.

find_runmod([{runmod, [Mod]} |_]) ->
    {ok,l2a(Mod)};
find_runmod([_|T]) ->
    find_runmod(T);
find_runmod([]) ->
    false.

l2a(L) when list(L) -> list_to_atom(L);
l2a(A) when atom(A) -> A.



%%----------------------------------------------------------------------
%% Func: init/1
%% Returns: {ok, State}          |
%%          {ok, State, Timeout} |
%%          ignore               |
%%          {stop, Reason}
%%----------------------------------------------------------------------
init([]) ->
    process_flag(trap_exit, true),
    put(start_time, calendar:local_time()),  %% for uptime
    {Debug, Trace, Conf, RunMod, Embed} = get_app_args(),
    case Embed of 
	false ->
	    case yaws_config:load(Conf, Trace, Debug) of
		{ok, Gconf, Sconfs} ->
		    erase(logdir),
		    ?Debug("Conf = ~p~n", [?format_record(Gconf, gconf)]),
		    yaws_log:setdir(Gconf#gconf.logdir, Sconfs),
		    case Gconf#gconf.trace of
			{true, What} ->
			    yaws_log:open_trace(What);
			_ ->
			    ok
		    end,
		    init2(Gconf, Sconfs, RunMod, true);
		{error, E} ->
		    case erase(logdir) of
			undefined ->
			    error_logger:error_msg("Yaws: Bad conf: ~p~n", 
						   [E]),
			    init:stop(),
			    {stop, E};
			Dir ->
			    yaws_log:setdir(Dir, []),
			    error_logger:error_msg("Yaws: bad conf: ~s~n",[E]),
			    init:stop(),
			    {stop, E}
		    end
	    end;
	true ->
	    init2(yaws_config:make_default_gconf(Debug), [], undef, true)
    end.


init2(Gconf, Sconfs, RunMod, FirstTime) ->

    lists:foreach(
      fun(D) ->
	      yaws_debug:format(Gconf, "Add path ~p~n", [D]),
	      code:add_pathz(D)
      end, Gconf#gconf.ebin_dir),


    %% start user provided application
    case RunMod of
	{ok,Mod} -> 
	    runmod(Mod);
	_ -> 
	    false
    end,
    lists:foreach(fun(M) -> runmod(M) end, Gconf#gconf.runmods),

    %% start the individual server processes
    L = lists:map(
	  fun(Group) ->
		  proc_lib:start_link(?MODULE, gserv, [Gconf, Group])
	  end, Sconfs),
    L2 = lists:zf(fun({error, F, A}) ->	
			  error_logger:error_msg(F, A),
			  exit(nostart);
		     ({error, Reason}) ->
			  error_logger:error_msg("FATAL: ~p~n", [Reason]),
			  exit(nostart);
		     ({_Pid, _SCs}) ->
			  true;
		     (none) ->
			  false
		  end, L),
    ?Debug("L=~p~n", [yaws_debug:nobin(L)]),
    if
	FirstTime == true ->
	    CTL = proc_lib:spawn_link(yaws_ctl, start, 
				      [self(), Gconf#gconf.uid]),
	    receive
		{CTL, ok} ->
		    ok;
		{CTL, {error, R}} ->
		    exit(R)
	    end;
		
	true ->
	    ok
    end,
    file:make_dir("/tmp/yaws"),
    set_writeable("/tmp/yaws"),
    case (catch yaws_log:uid_change(Gconf)) of
	ok ->
	    ok;
	ERR ->
	    error_logger:error_msg("Failed to change uid on logfiles:~n~p",
				   [ERR]),
	    exit(error)
    end,

    %% and now finally, we've opened the ctl socket and are
    %% listening to all sockets we can possibly change username
 

    GC2 = case (catch yaws:setuser(Gconf#gconf.username)) of
	      ignore ->
		  Gconf;
	      {ok, NewUid} ->
		  error_logger:info_msg("Changed uid to ~s~n", [NewUid]),
		  Gconf#gconf{uid = NewUid};
	      Other ->
		  error_logger:error_msg("Failed to set user:~n~p", [Other]),
		  exit(Other)
	  end,
    case (catch setup_tdir(GC2)) of
	ok ->
	    ok;
	_Error ->
	    exit(error)
    end,
    lists:foreach(fun({Pid, _}) -> Pid ! {newuid, GC2#gconf.uid} end, L2),
    {ok, {GC2, L2, 0}}.


%%----------------------------------------------------------------------
%% Func: handle_call/3
%% Returns: {reply, Reply, State}          |
%%          {reply, Reply, State, Timeout} |
%%          {noreply, State}               |
%%          {noreply, State, Timeout}      |
%%          {stop, Reason, Reply, State}   | (terminate/2 is called)
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_call(status, _From, State) ->
    Reply = {State, get(start_time)},
    {reply, Reply, State};
handle_call(pids, _From, State) ->  %% for gprof
    L = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    {reply, [self() | L], State};

handle_call(mnum, _From, {GC, Group, Mnum}) ->
    {reply, Mnum+1,   {GC, Group, Mnum+1}};

handle_call({setconf, GC, Groups}, _From, State) ->
    %% First off, terminate all currently running processes
    Curr = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    lists:foreach(fun(Pid) ->
			  Pid ! {self(), stop},
			  receive
			      {Pid, ok} ->
				  ok
			  end
		  end, Curr),
    case init2(GC, Groups, undef, false) of
	{ok, State2} ->
	    {reply, ok, State2};
	Err ->
	    {reply, Err, {GC, [], 0}}
    end;

handle_call(getconf, _From, State) ->
    {GC, Pairs, _} = State,
    Groups = lists:map(fun({_Pid, SCs}) -> SCs end, Pairs),
    {reply, {ok, GC, Groups}, State}.
			       
			      



%%----------------------------------------------------------------------
%% Func: handle_cast/2
%% Returns: {noreply, State}          |
%%          {noreply, State, Timeout} |
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_cast(_Msg, State) ->
    {noreply, State}.

%%----------------------------------------------------------------------
%% Func: handle_info/2
%% Returns: {noreply, State}          |
%%          {noreply, State, Timeout} |
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_info({'EXIT', Pid, Reason},  State) ->
    L = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    case lists:member(Pid, L) of
	true ->
	    %% one of our gservs died
	    error_logger:format("yaws: FATAL gserv died ~p~n", [Reason]),
	    exit(restartme);
	false ->
	    ignore
    end,
    {noreply, State};


handle_info(_Msg, State) ->
    ?Debug("GOT ~p~n", [_Msg]),
    {noreply, State}.

%%----------------------------------------------------------------------
%% Func: terminate/2
%% Purpose: Shutdown the server
%% Returns: any (ignored by gen_server)
%%----------------------------------------------------------------------
terminate(_Reason, _State) ->
    ok.


%% we search and setup www authenticate for each directory
%% specified as an auth directory. These are merged with server conf.

setup_auth(SC) ->
    ?Debug("setup_auth(~p)", [yaws_debug:nobin(SC)]),
    ?f(lists:map(fun(Auth) ->
		      add_yaws_auth(Auth#auth.dir, Auth)
	      end, SC#sconf.authdirs)).

add_yaws_auth(Dirs, A) ->
    lists:map(
      fun(Dir) ->
	  case file:consult([Dir, [$/|".yaws_auth"]]) of
	      {ok, [{realm, Realm} |TermList]} ->
		  {Dir, A#auth{realm = Realm,
			       users = TermList++A#auth.users}};
	      {ok, TermList} ->
		  {Dir, A#auth{users = TermList++A#auth.users}};
	      {error, enoent} ->
		  {Dir, A};
	      _Err ->
		  error_logger:format("Bad .yaws_auth file in dir ~p~n", [Dir]),
		  {Dir, A}
	  end
      end, Dirs).


do_listen(SC) ->
    case SC#sconf.ssl of
	undefined ->
	    {nossl, gen_tcp:listen(SC#sconf.port, opts(SC))};
	SSL ->
	    {ssl, ssl:listen(SC#sconf.port, ssl_opts(SC, SSL))}
    end.

set_writeable(Dir) ->
    set_dir_mode(Dir, 8#777).

set_dir_mode(Dir, Mode) ->
    {ok, FI} = file:read_file_info(Dir),
    file:write_file_info(Dir, FI#file_info{mode = Mode}).


gserv(_, []) ->
    proc_lib:init_ack(none);

%% One server per IP we listen to
gserv(GC, Group0) ->
    process_flag(trap_exit, true),
    ?TC([{record, GC, gconf}]),
    Group = map(fun(SC) -> 
			E = ets:new(yaws_code, [public, set]),
			ets:insert(E, {num_files, 0}),
			ets:insert(E, {num_bytes, 0}),
			Auth = setup_auth(SC),
			SC#sconf{ets = E,
				 authdirs = Auth}
		end, Group0),
    SC = hd(Group),
    case do_listen(SC) of
	{SSLBOOL, {ok, Listen}} ->
	    call_start_mod(SC),
	    error_logger:info_msg(
	      "Yaws: Listening to ~s:~w for servers~s~n",
	      [yaws:fmt_ip(SC#sconf.listen),
	       SC#sconf.port,
	       catch map(fun(S) ->  
				 io_lib:format("~n - ~s under ~s",
					       [S#sconf.servername,
						S#sconf.docroot])
			 end, Group)
	      ]),
	    proc_lib:init_ack({self(), Group}),
	    N = receive
		    {newuid, UID} ->
			UID
		end,
	    GS = #gs{gconf = GC#gconf{uid = N},
		     group = Group,
		     ssl = SSLBOOL,
		     l = Listen},
	    acceptor(GS),
	    gserv(GS, [], 0);
	{_,Err} ->
	    error_logger:format("Yaws: Failed to listen ~s:~w  : ~p~n",
				[yaws:fmt_ip(SC#sconf.listen),
				 SC#sconf.port, Err]),
	    proc_lib:init_ack({error, "Can't listen to socket: ~p ",[Err]}),
	    exit(normal)
    end.
			

setup_tdir(GC) ->
    Tdir = "/tmp/yaws/" ++ GC#gconf.uid,
    file:make_dir(Tdir),
    set_dir_mode(Tdir, 8#700),
    case file:list_dir(Tdir) of
	{ok, Files0} ->
	    case lists:zf(
		   fun(F) -> 
			   Fname = Tdir ++ "/" ++ F,
			   case file:delete(Fname) of
			       ok ->
				   false;
			       {error, Reason} ->
				   {true, {Fname, Reason}}
			   end
		   end, Files0) of
		[] ->
		    ok;
		[{FF, RR}|_] ->
		    error_logger:format("Failed to delete ~p~n~p", [FF, RR]),
		    {error, RR}
	    end;
	{error, Reason} ->
	    error_logger:format("Failed to list ~p probably "
				"due to permission errs: ~p",
				[Tdir, Reason]),
	    {error, Reason}
    end.




gserv(GS, Ready, Rnum) ->
    receive
	{From , status} ->
	    From ! {self(), GS},
	    gserv(GS, Ready, Rnum);
	{From, next} when Ready == [] ->
	    acceptor(GS),
	    gserv(GS, Ready, Rnum);
	{From, next} ->
	    [R|RS] = Ready,
	    R ! {self(), accept},
	    gserv(GS, RS, Rnum-1);
	{From, done_client, Int} ->
	    GS2 = GS#gs{sessions = GS#gs.sessions + 1,
			reqs = GS#gs.reqs + Int},
	    if
		Rnum == 8 ->
		    From ! {self(), stop},
		    gserv(GS2, Ready, Rnum);
		Rnum < 8 ->
		    gserv(GS2, [From | Ready], Rnum+1)
	    end;
	{'EXIT', _Pid, _} ->
	    gserv(GS, Ready, Rnum);
	{From, stop} ->   
	    unlink(From),
	    {links, Ls} = process_info(self(), links),
	    lists:foreach(fun(X) -> unlink(X), exit(X, kill) end, Ls),
	    From ! {self(), ok},
	    exit(normal)
    end.
	    
 
call_start_mod(SC) ->
    case SC#sconf.start_mod of
	undefined ->
	    ok;
	Mod0 ->
	    Mod = l2a(Mod0),
	    case code:ensure_loaded(Mod) of
		{module, Mod} ->
		    spawn(Mod, start, [SC]);
		Err ->
		    error_logger:format("Cannot load module ~p: ~p~n", 
					[Mod,Err])
	    end
    end.



opts(SC) ->
    [binary, 
     {ip, SC#sconf.listen},
     {packet, http},
     {recbuf, 8192},
     {reuseaddr, true},
     {active, false}
    ].



ssl_opts(SC, SSL) ->
    Opts = [
	    binary,
	    {ip, SC#sconf.listen},
	    {packet, 0},
	    {active, false} | ssl_opts(SSL)],
    ?Debug("SSL opts  ~p~n", [Opts]),
    Opts.




ssl_opts(SSL) ->
    L = [if SSL#ssl.keyfile /= undefined ->
		 {keyfile, SSL#ssl.keyfile};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.certfile /= undefined ->
		 {certfile, SSL#ssl.certfile};
	    true ->
		 false
	 end,

	 if SSL#ssl.cacertfile /= undefined ->
		 {cacertfile, SSL#ssl.cacertfile};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.verify /= undefined ->
		 {verify, SSL#ssl.verify};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.password /= undefined ->
		 {password, <PASSWORD>};
	    true ->
		 false
	 end,
	 if SSL#ssl.ciphers /= undefined ->
		 {ciphers, SSL#ssl.ciphers};
	    true ->
		 false
	 end
	],
    filter_false(L).


filter_false(L) ->
    [X || X <- L, X /= false].

	   
	 
do_accept(GS) when GS#gs.ssl == nossl ->
    gen_tcp:accept(GS#gs.l);
do_accept(GS) when GS#gs.ssl == ssl ->
    ssl:accept(GS#gs.l).


acceptor(GS) ->
    proc_lib:spawn_link(yaws_server, acceptor0, [GS, self()]).
acceptor0(GS, Top) ->
    ?TC([{record, GS, gs}]),
    X = do_accept(GS),
    ?Debug("do_accept ret: ~p~n", [X]),
    Top ! {self(), next},
    case X of
	{ok, Client} ->
	    case (GS#gs.gconf)#gconf.trace of  %% traffic trace
		{true, _} ->
		    {ok, {IP, Port}} = case GS#gs.ssl of
					   ssl ->
					       ssl:peername(Client);
					   nossl ->
					       inet:peername(Client)
				       end,
		    Str = ?F("New (~p) connection from ~s:~w~n", 
			     [GS#gs.ssl, yaws:fmt_ip(IP),Port]),
		    yaws_log:trace_traffic(from_client, Str);
		_ ->
		    ok
	    end,

	    Res = (catch aloop(Client, GS,  0)),
	    ?Debug("RES = ~p~n", [Res]),
	    if
		GS#gs.ssl == nossl ->
		    gen_tcp:close(Client);
		GS#gs.ssl == ssl ->
		    ssl:close(Client)
	    end,
	    case Res of
		{ok, Int} when integer(Int) ->
		    Top ! {self(), done_client, Int};
		{'EXIT', normal} ->
		    exit(normal);
		{'EXIT', {badarg, [{erlang, apply, _} |_]}} ->
		    %% hopeless cracker request
		    %% don't crash log
		    exit(normal);
		{'EXIT', Reason} ->
		    error_logger:error_msg("Yaws process died: ~p~n", [Reason]),
		    exit(normal)
	    end,
	    %% we cache processes
	    receive
		{Top, stop} ->
		    exit(normal);
		{Top, accept} ->
		    acceptor0(GS, Top)
	    end;
	_ ->
	    exit(normal)
    end.
     


%%%----------------------------------------------------------------------
%%% Internal functions
%%%----------------------------------------------------------------------

aloop(CliSock, GS, Num) when GS#gs.ssl == nossl ->
    ?TC([{record, GS, gs}]),
    case http_get_headers(CliSock, GS#gs.gconf) of
	done ->
	    {ok, Num};
	{Req, H} ->
	    SC = pick_sconf(GS#gs.gconf, H, GS#gs.group),
	    ?Debug("SC: ~p", [?format_record(SC, sconf)]),
	    ?TC([{record, SC, sconf}]),
	    ?Debug("Headers = ~p~n", [?format_record(H, headers)]),
	    ?Debug("Request = ~p~n", [?format_record(Req, http_request)]),
	    IP = case SC#sconf.access_log of
		     true ->
			 {ok, {Ip, _Port}} = inet:peername(CliSock),
			 Ip;
		     _ ->
			 undefined
		 end,
	    put(outh, #outh{}),
	    Res = apply(yaws_server, Req#http_request.method, 
			[CliSock, GS#gs.gconf, SC, Req, H]),
	    maybe_access_log(IP, SC, Req, H),
	    case Res of
		continue ->
		    aloop(CliSock, GS, Num+1);
		done ->
		    {ok, Num+1};
		{page, Page} ->
		    put(outh, #outh{}),
		    apply(yaws_server, Req#http_request.method, 
			  [CliSock, GS#gs.gconf, SC#sconf{appmods=[]}, 
			   Req#http_request{path = {abs_path, Page}}, H])
	    end
    end;



aloop(CliSock, GS, Num) when GS#gs.ssl == ssl ->
    ?TC([{record, GS, gs}]),
    case yaws_ssl:ssl_get_headers(CliSock, GS#gs.gconf) of
	done ->
	    {ok, Num};
	{ok, Req, H, Trail} ->
	    put(ssltrail, Trail),  %% hack hack hack
	    SC = hd(GS#gs.group),
	    IP = case SC#sconf.access_log of
		     true ->
			 {ok, {Ip, _Port}} = ssl:peername(CliSock),
			 Ip;
		     _ ->
			 undefined
		 end,
	    put(outh, #outh{}),
	    Res = apply(yaws_server, Req#http_request.method, 
			[CliSock, GS#gs.gconf, SC, Req, H]),
	    maybe_access_log(IP, SC, Req, H),
	    case Res of
		continue ->
		    aloop(CliSock, GS, Num+1);
		done ->
		    {ok, Num+1};
		{page, Page} ->
		    put(outh, #outh{}),
		    apply(yaws_server, Req#http_request.method, 
			  [CliSock, GS#gs.gconf, SC#sconf{appmods = []}, 
			   Req#http_request{path = {abs_path, Page}}, H])
	    end
    end.



pick_sconf(GS, H, Group) ->
    case H#headers.host of
	undefined ->
	    hd(Group);
	Host ->
	    pick_host(GS, Host, Group, Group)
    end.


pick_host(_GC, Host, [H|_T], _Group) when H#sconf.servername == Host ->
    H;
pick_host(GC, Host, [_|T], Group) ->
    pick_host(GC, Host, T, Group);
pick_host(_GC, _Host, [], Group) ->
    hd(Group).


inet_peername(Sock, SC) ->
    case SC#sconf.ssl of
	undefined ->
	    inet:peername(Sock);
	_SSL ->
	    ssl:peername(Sock)
    end.
	    

maybe_access_log(Ip, SC, Req, H) ->
    ?TC([{record, SC, sconf}]),
    Status = case yaws:outh_get_status_code() of
		 undefined -> "-";
		 I -> integer_to_list(I)
	     end,
    Len = case yaws:outh_get_contlen() of
	      undefined ->
		  "-";
	      I2 -> integer_to_list(I2)
	  end,
    Ver = case Req#http_request.version of
	      {1,0} ->
		  "HTTP/1.0";
	      {1,1} ->
		  "HTTP/1.1";
	      {0,9} ->
		  "HTTP/0.9" 
	  end,
    case SC#sconf.access_log of
	true ->
	    Path = safe_decode_path(Req#http_request.path),
	    Meth = atom_to_list(Req#http_request.method),
	    Referrer = optional_header(H#headers.referer),
	    UserAgent = optional_header(H#headers.user_agent),
	    yaws_log:accesslog(SC#sconf.servername, Ip, 
			       [Meth, $\s, Path, $\s, Ver], 
			       Status, Len, Referrer, UserAgent);
	false ->
	    ignore
    end.

safe_decode_path(Path) ->
    case (catch decode_path(Path)) of
	{'EXIT', _} ->
	    "/undecodable_path";
	Val ->
	    Val 
    end.


decode_path({abs_path, Path}) ->
    yaws_api:url_decode(Path).

optional_header(Item) ->
    case Item of
	undefined -> "-";
	Item -> Item
    end.

do_recv(Sock, Num, TO, nossl) ->
    gen_tcp:recv(Sock, Num, TO);
do_recv(Sock, Num, _TO, ssl) ->
    case erase(ssltrail) of %% hack from above ...
	undefined ->
	    split_recv(ssl:recv(Sock, 0), Num);   %% ignore Num val ??? TO ??
	Bin ->
	    {ok, Bin}
    end.

%% weird ... ssl module doesn't respect Num arg for binary socks
split_recv({ok, B}, Num) when integer(Num) ->
    case B of
	<<Bin:Num/binary , Tail/binary>> ->
	    put(ssltrail, Tail),
	    {ok, Bin};
	_ ->
	    {ok, B}
    end;
split_recv(E, _) ->
    E.


	
	 

cli_recv(S, Num, GC, SslBool) when GC#gconf.trace == false ->
    do_recv(S, Num, GC#gconf.timeout, SslBool);
cli_recv(S, Num, GC, SslBool) ->
    Res = do_recv(S, Num, GC#gconf.timeout, SslBool),
    case Res of
	{ok, Val} when tuple(Val) ->
	    yaws_log:trace_traffic(from_client, ?F("~p~n", [Val]));
	{error, What} ->
	    yaws_log:trace_traffic(from_client, ?F("~n~p~n", [What]));
	{ok, http_eoh} ->
	    ok;
	{ok, Val} when GC#gconf.trace == {true, traffic} ->
	    yaws_log:trace_traffic(from_client, Val);
	{ok, Val} ->
	    {ok, Val}
    end,
    Res.



gen_tcp_send(S, Data, SC, GC) ->
    Res = case SC#sconf.ssl of
	      undefined ->
		  gen_tcp:send(S, Data);
	      _SSL ->
		  ssl:send(S, Data)
	  end,
    case GC#gconf.debug of
	false ->
	    case Res of
		ok ->
		    ok;
		Err ->
		    exit(Err)
	    end;
	true ->
	    case Res of
		ok ->
		    ?Debug("Sent ~p~n", [yaws_debug:nobin(Data)]),
		    ok;
		Err ->
		    {B2, Size} = strip(Data),
		    yaws_debug:derror(GC, "Failed to send ~w bytes:~n~p "
				      "on socket ~p: ~p~n",
				      [Size, B2, S, Err]),
		    exit(Err)
	    end
    end.


strip(Data) ->
    L = list_to_binary([Data]),
    case L of
	<<Head:50/binary, _/binary>> ->
	    {binary_to_list(<<Head/binary, ".....">>), size(L)};
	_ ->
	    {binary_to_list(L), size(L)}
    end.
	 


http_get_headers(CliSock, GC) ->
    ?TC([{record, GC, gconf}]),
    inet:setopts(CliSock, [{packet, http}]),
    R = http_recv_request(CliSock, GC),
    H = http_get_headers(CliSock, R, GC, #headers{}),
    {R, H}.


http_recv_request(CliSock, GC) ->
    case cli_recv(CliSock, 0, GC, nossl) of
	{ok, R} when element(1, R) == http_request ->
	    R;
	{error, {http_error, "\r\n"}} ->
	    http_recv_request(CliSock, GC);
	{error, {http_error, "\n"}} ->
	    http_recv_request(CliSock, GC);
	_Other ->
	    ?Debug("Got ~p~n", [_Other]),
	    exit(normal)
    end.


		  
http_get_headers(CliSock, Req, GC, H) ->
    ?TC([{record, GC, gconf}]),
    case cli_recv(CliSock, 0, GC, nossl) of
	{ok, {http_header,  _Num, 'Host', _, Host}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{host = Host});
	{ok, {http_header, _Num, 'Connection', _, Conn}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{connection = Conn});
	{ok, {http_header, _Num, 'Accept', _, Accept}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{accept = Accept});
	{ok, {http_header, _Num, 'If-Modified-Since', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			     H#headers{if_modified_since = X});
	{ok, {http_header, _Num, 'If-Match', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_match = X});
	{ok, {http_header, _Num, 'If-None-Match', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_none_match = X});
	{ok, {http_header, _Num, 'If-Range', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_range = X});
	{ok, {http_header, _Num, 'If-Unmodified-Since', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{if_unmodified_since = X});
	{ok, {http_header, _Num, 'Range', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{range = X});
	{ok, {http_header, _Num, 'Referer',_, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{referer = X});
	{ok, {http_header, _Num, 'User-Agent', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{user_agent = X});
	{ok, {http_header, _Num, 'Accept-Ranges', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{accept_ranges = X});
	{ok, {http_header, _Num, 'Cookie', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{cookie = [X|H#headers.cookie]});
	{ok, {http_header, _Num, 'Keep-Alive', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{keep_alive = X});
	{ok, {http_header, _Num, 'Content-Length', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{content_length = X});
	{ok, {http_header, _Num, 'Content-Type', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{content_type = X});
	{ok, {http_header, _Num, 'Authorization', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{authorization = parse_auth(X)});
	{ok, http_eoh} ->
	    H;
	{error, {http_error, "\r\n"}} ->
	     http_get_headers(CliSock, Req, GC, H);
	{error, {http_error, "\n"}} ->
	     http_get_headers(CliSock, Req, GC, H);
	{ok, X} ->
	    ?Debug("OTHER header ~p~n", [X]),
	    http_get_headers(CliSock, Req, GC, 
			     H#headers{other=[X|H#headers.other]});
	_Err ->
	    exit(normal)
    
    end.


inet_setopts(SC, S, Opts) when SC#sconf.ssl == undefined ->
    inet:setopts(S, Opts);
inet_setopts(_,_,_) ->
    ok.  %% noop for ssl


%% ret:  continue | done
'GET'(CliSock, GC, SC, Req, Head) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("GET ~p", [?format_record(Req, http_request)]),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    flush(SC, CliSock, Head#headers.content_length),
    ARG = make_arg(CliSock, Head, Req, GC, SC),
    ARG2 = apply(SC#sconf.arg_rewrite_mod, arg_rewrite, [ARG]),
    handle_request(CliSock, GC, SC, ARG2, 0).


'POST'(CliSock, GC, SC, Req, Head) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("POST Req=~p H=~p~n", [?format_record(Req, http_request),
				  ?format_record(Head, headers)]),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    PPS = SC#sconf.partial_post_size,
    Bin = case Head#headers.content_length of
	      undefined ->
		  case Head#headers.connection of
		      "close" ->
			  get_client_data(CliSock, all, GC,
					  is_ssl(SC#sconf.ssl));
		      _ ->
			  ?Debug("No content length header ",[]),
			  exit(normal)
		  end;
	      Len when integer(PPS) ->
		  Int_len = list_to_integer(Len),
		  if 
		      Int_len == 0 ->
			  <<>>;
		      PPS < Int_len ->
			  {partial, get_client_data(CliSock, PPS, GC,
						    is_ssl(SC#sconf.ssl))};
		      true ->
			  get_client_data(CliSock, Int_len, GC, 
					  is_ssl(SC#sconf.ssl))
		  end;
	      Len when PPS == nolimit ->
		  Int_len = list_to_integer(Len),
		  if
		      Int_len == 0 ->
			  <<>>;
		      true ->
			  get_client_data(CliSock, Int_len, GC, 
					  is_ssl(SC#sconf.ssl))
		  end
	  end,
    ?Debug("POST data = ~s~n", [binary_to_list(un_partial(Bin))]),
    ARG = make_arg(CliSock, Head, Req, GC, SC),
    ARG2 = ARG#arg{clidata = Bin},
    ARG3 = apply(SC#sconf.arg_rewrite_mod, arg_rewrite, [ARG2]),
    handle_request(CliSock, GC, SC, ARG3, size(un_partial(Bin))).


is_ssl(undefined) ->
    nossl;
is_ssl(R) when record(R, ssl) ->
    ssl.

un_partial({partial, Bin}) ->
    Bin;
un_partial(Bin) ->
    Bin.

%% will throw
'HEAD'(CliSock, GC, SC, Req, Head) ->
    'GET'(CliSock, GC, SC, Req, Head).

'TRACE'(_CliSock, _GC, _SC, _Req, _Head) ->
    nyi.

'OPTIONS'(CliSock, GC, SC, Req, Head) ->
    ?Debug("OPTIONS", []),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    flush(SC, CliSock, Head#headers.content_length),
    _ARG = make_arg(CliSock, Head, Req, GC, SC),
    ?Debug("OPTIONS delivering", []),
    deliver_options(CliSock, Req, GC, SC).

make_arg(CliSock, Head, Req, _GC, SC) ->
    ?TC([{record, _GC, gconf}, {record, SC, sconf}]),
    #arg{clisock = CliSock,
	 headers = Head,
	 req = Req,
	 opaque = SC#sconf.opaque,
	 pid = self(),
	 docroot = SC#sconf.docroot}.

handle_request(CliSock, GC, SC, ARG, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    Req = ARG#arg.req,
    H = ARG#arg.headers,
    {abs_path, RawPath} = Req#http_request.path,
    case (catch yaws_api:url_decode_q_split(RawPath)) of
	{'EXIT', _} ->   %% weird broken cracker requests
	    deliver_403(CliSock, Req, GC, SC);
	{DecPath, QueryPart} ->
	    UT =  url_type(GC, SC, DecPath, QueryPart),
	    ARG2 = ARG#arg{fullpath=UT#urltype.fullpath},
	    case SC#sconf.authdirs of
		[] ->
		    handle_ut(CliSock, GC, SC, Req, H, ARG2, UT, N);
		_Adirs ->
		    %% we have authentication enabled, check auth
		    UT2 = unflat(UT),
		    case is_authenticated(SC, UT2, Req, H) of
			true ->
			    handle_ut(CliSock, GC, SC, Req, H, ARG2, UT2, N);
			{false, Realm} ->
			    deliver_401(CliSock, Req, GC, Realm, SC)
		    end
	    end
    end.


unflat(U) ->
    U2 = case U#urltype.path of
	     {noflat, F} ->
		 U#urltype{path = lists:flatten(F)};
	     _ ->
		 U
	 end,
    U2.

is_authdir(Req_dir, [{Auth_dir, Auth}|T]) ->
    case lists:prefix(Auth_dir, Req_dir) of
	true ->
	    {true, Auth};
	false ->
	    is_authdir(Req_dir, T)
    end;
is_authdir(_, []) ->
    false.

is_authenticated(SC, UT, _Req, H) ->
    case is_authdir(UT#urltype.path, SC#sconf.authdirs) of
	{true, #auth{dir = _Dir, realm = Realm, users = Users}} ->
	    case H#headers.authorization of
		undefined ->
		    {false, Realm};
		{User, Password} ->
		    case lists:member({User, Password}, Users) of
			true ->
			    true;
			false ->
			    {false, "Got username"}
		    end
	    end;
	false ->
	    true
    end.


handle_ut(CliSock, GC, SC, Req, H, ARG, UT, N) ->
    case UT#urltype.type of
	error ->
	    A2 = ARG#arg{querydata = UT#urltype.q},
	    yaws:outh_set_dyn_headers(Req, H),
	    do_appmod(SC#sconf.errormod_404, out404, CliSock, GC, SC, 
		      Req, [A2, GC, SC], UT, N);
	directory when SC#sconf.dir_listings == true ->
	    P = UT#urltype.dir,
	    yaws:outh_set_dyn_headers(Req, H),
	    yaws_ls:list_directory(CliSock, UT#urltype.data, P, Req, GC, SC);
	directory ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_403(CliSock, Req, GC, SC);
	regular ->
	    ETag = yaws:make_etag(UT#urltype.finfo),
	    case H#headers.if_none_match of
		undefined ->
		    yaws:outh_set_static_headers(Req, UT, H),
		    deliver_file(CliSock, GC, SC, Req, UT);
		ETag ->
		    yaws:outh_set_304_headers(Req, UT, H),
		    deliver_accumulated(CliSock, GC, SC),
		    case yaws:outh_get_doclose() of
			true  -> done;
			false -> continue
		    end;
		_ ->
		    yaws:outh_set_static_headers(Req, UT, H),
		    deliver_file(CliSock, GC, SC, Req, UT)
	    end;
	yaws ->
	    yaws:outh_set_dyn_headers(Req, H),
	    do_yaws(CliSock, GC, SC, Req,  
		    [ARG#arg{querydata = UT#urltype.q}], UT, N);
	forbidden ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_403(CliSock, Req, GC, SC);
	redir_dir ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_302(CliSock, Req, GC, SC, ARG);
	appmod ->
	    yaws:outh_set_dyn_headers(Req, H),
	    close_if_HEAD(Req, 
			  fun() ->
				  deliver_accumulated(CliSock, GC,SC),
				  throw({ok, 1}) 
			  end),
	    {Mod, PathData} = UT#urltype.data,
	    A2 = ARG#arg{appmoddata = PathData,
			 querydata = UT#urltype.q,
			 appmod_prepath = UT#urltype.path},
	    do_appmod(Mod, out, CliSock, GC, SC, Req,  [A2], UT, N)
    end.

	
parse_auth("Basic " ++ Auth64) ->
    case decode_base64(Auth64) of
	{error, _Err} ->
	    undefined;
	Auth ->
	    case string:tokens(Auth, ":") of
		[User, Pass] ->
		    {User, Pass};
		_ ->
		    undefined
	    end
    end;
parse_auth(_) ->
    undefined.




%% we may have content, 

new_redir_h(OH, Loc) ->
    OH2 = OH#outh{status = 302,
		  location = Loc},
    put(outh, OH2).



% we must deliver a 302 if the browser asks for a dir
% without a trailing / in the HTTP req
% otherwise the relative urls in /dir/index.html will be broken.


deliver_302(CliSock, Req, GC, SC, Arg) ->
    ?Debug("in redir 302 ",[]),
    H = get(outh),

    Scheme = redirect_scheme(SC),
    Headers = Arg#arg.headers,
    DecPath = decode_path(Req#http_request.path),
    Loc = case string:tokens(DecPath, "?") of
	      [P] ->
		  ["Location: ", Scheme,
		   Headers#headers.host, P, "/\r\n"];
	      [P, Q] ->
		  ["Location: ", Scheme,
		   Headers#headers.host, P, "/", Q, "\r\n"]
	  end,
    
    Ret = case yaws:outh_get_chunked() of
	      true ->
		  accumulate_content([crnl(), "0", crnl2()]),
		  continue;
	      false ->
		  done
	  end,

    new_redir_h(H, Loc),
    deliver_accumulated(CliSock, GC, SC),
    Ret.


redirect_scheme(SC) ->
    case {SC#sconf.ssl,SC#sconf.rmethod} of
	{_, Method} when list(Method) ->
	    Method++"://";
	{undefined,_} ->
	    "http://";
	{_SSl,_} ->
	    "https://"
    end.    

redirect_host(SC, HostHdr) ->
    case SC#sconf.rhost of
	undefined ->
	    HostHdr;
	_ ->
	    SC#sconf.rhost
    end.

redirect_port(SC) ->
    case {SC#sconf.ssl, SC#sconf.port} of
	{undefined, 80} ->
               "";
	{undefined, Port} -> 
               [$:|integer_to_list(Port)];
	{_SSL, 443} ->
               "";
	{_SSL, Port} -> 
               [$:|integer_to_list(Port)]
    end.    

redirect_scheme_port(SC) ->
    Scheme = redirect_scheme(SC),
    PortPart = redirect_port(SC),
    {Scheme, PortPart}.

servername_sans_port(Servername) ->
    case string:chr(Servername, $:) of
	0 ->
	    Servername;
	N ->
	    lists:sublist(Servername, N-1)
    end.

deliver_options(CliSock, _Req, GC, SC) ->
    H = #outh{status = 200,
	      doclose = false,
	      chunked = false,
	      server = yaws:make_server_header(),
	      allow = yaws:make_allow_header()},
    put(outh, H),
    deliver_accumulated(CliSock, GC, SC),
    continue.

deliver_401(CliSock, _Req, GC, Realm, SC) ->
    B = list_to_binary("<html> <h1> 401 authentication needed  "
		       "</h1></html>"),
    H = #outh{status = 401,
	      doclose = true,
	      chunked = false,
	      server = yaws:make_server_header(),
	      connection = yaws:make_connection_close_header(true),
	      content_length = yaws:make_content_length_header(size(B)),
	      www_authenticate = yaws:make_www_authenticate_header(Realm),
	      contlen = size(B),
	      content_type = yaws:make_content_type_header("text/html")},
    put(outh, H),
    accumulate_content(B),
    deliver_accumulated(CliSock, GC, SC),
    done.
    




deliver_403(CliSock, _Req, GC, SC) ->
    B = list_to_binary("<html> <h1> 403 Forbidden</h1> </html>"),
    H = #outh{status = 403,
	      doclose = true,
	      chunked = false,
	      server = yaws:make_server_header(),
	      connection = yaws:make_connection_close_header(true),
	      content_length = yaws:make_content_length_header(size(B)),
	      contlen = size(B),
	      content_type = yaws:make_content_type_header("text/html")},
    put(outh, H),
    accumulate_content(B),
    deliver_accumulated(CliSock, GC, SC),
    done.
    



do_yaws(CliSock, GC, SC, Req, ARG, UT, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    FileAtom = list_to_atom(UT#urltype.fullpath),
    Mtime = mtime(UT#urltype.finfo),
    case ets:lookup(SC#sconf.ets, FileAtom) of
	[{FileAtom, spec, Mtime1, Spec, Es}] when Mtime1 == Mtime,
						  Es == 0 ->
	    deliver_dyn_file(CliSock, GC, SC, Req, Spec, ARG, UT, N);
	Other  ->
	    del_old_files(Other),
	    {ok, [{errors, Errs}| Spec]} = 
		yaws_compile:compile_file(UT#urltype.fullpath, GC, SC),
	    ?Debug("Spec for file ~s is:~n~p~n",[UT#urltype.fullpath, Spec]),
	    ets:insert(SC#sconf.ets, {FileAtom, spec, Mtime, Spec, Errs}),
	    deliver_dyn_file(CliSock, GC, SC, Req, Spec, ARG, UT, N)
    end.


del_old_files([]) ->
    ok;
del_old_files([{_FileAtom, spec, _Mtime1, Spec, _}]) ->
    lists:foreach(
      fun({mod, _, _, _,  Mod, _Func}) ->
	      F="/tmp/yaws/" ++ yaws:to_list(Mod) ++ ".erl",
	      code:purge(Mod),
	      code:purge(Mod),
	      file:delete(F);
	 (_) ->
	      ok
      end, Spec).
		     

get_client_data(CliSock, all, GC, SSlBool) ->
    get_client_data(CliSock, all, cli_recv(CliSock, 4000, GC, SSlBool), 
		    GC, SSlBool);

get_client_data(CliSock, Len, GC, SSlBool) ->
    case cli_recv(CliSock, Len, GC, SSlBool) of
	{ok, B} when size(B) == Len ->
	    B;
	_Other ->
	    ?Debug("get_client_data: ~p~n", [_Other]),
	    exit(normal)
    end.

get_client_data(CliSock, all, {ok, B}, GC, SSlBool) ->
    B2 = get_client_data(CliSock, all, 
			 cli_recv(CliSock, 4000, GC, SSlBool), SSlBool),
    <<B/binary, B2/binary>>;
get_client_data(_CliSock, all, eof, _GC, _) ->
    <<>>.



do_appmod(Mod, FunName, CliSock, GC, SC, Req, ARG, UT, N) ->
    case yaws_call(0, "appmod", Mod, FunName, ARG, GC,SC, N) of
	{streamcontent, MimeType, FirstChunk} ->
	    yaws:outh_set_content_type(MimeType),
	    accumulate_chunk(FirstChunk),
	    deliver_accumulated(CliSock, GC, SC),
	    stream_loop(CliSock, GC, SC),
	    case yaws:outh_get_chunked() of
		true ->
		    gen_tcp_send(CliSock, [crnl(), "0", crnl2()], SC, GC),
		    continue;
		false ->
		    done
	    end;

	{streamcontent_with_size, Sz, MimeType, FirstChunk} ->
	    yaws:outh_set_content_type(MimeType),
	    yaws:outh_set_transfer_encoding_off(),
	    yaws:outh_set_content_length(Sz),
	    accumulate_chunk(FirstChunk),
	    deliver_accumulated(CliSock, GC, SC),
	    stream_loop(CliSock, GC, SC),
	    done;


	_ ->
	    %% finish up
	    deliver_dyn_file(CliSock, GC,SC,Req, UT, [], [], [], ARG,N)
    end.




%% do the header and continue
deliver_dyn_file(CliSock, GC, SC, Req, Specs, ARG, UT, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    Fd = ut_open(UT),
    Bin = ut_read(Fd),
    close_if_HEAD(Req, fun() ->
			       deliver_accumulated(CliSock,GC,SC),
			       do_tcp_close(CliSock, SC), 
			       throw({ok, 1}) 
		       end),
    deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin, Fd, Specs, ARG, N).



deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin, Fd, [H|T],ARG,N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}, {record, UT,urltype}]),
    ?Debug("deliver_dyn_file: ~p~n", [H]),
    case H of
	{mod, LineNo, YawsFile, NumChars, Mod, out} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    case yaws_call(LineNo, YawsFile, Mod, out, ARG, GC,SC, N) of
		break ->
		    deliver_dyn_file(CliSock, GC, SC, Req,  
				     UT, Bin, Fd, [],ARG,N) ;
		{page, Page} ->
		    {page, Page};
		{streamcontent, MimeType, FirstChunk} ->
		    yaws:outh_set_content_type(MimeType),
		    accumulate_chunk(FirstChunk),
		    deliver_accumulated(CliSock, GC, SC),
		    stream_loop(CliSock, GC, SC),
		    case yaws:outh_get_chunked() of
			true ->
			    gen_tcp_send(CliSock, [crnl(), "0", 
						   crnl2()], SC, GC),
			    continue;
			false ->
			    done
		    end;
		{streamcontent_with_size, Sz, MimeType, FirstChunk} ->
		    yaws:outh_set_content_type(MimeType),
		    yaws:outh_set_transfer_encoding_off(),
		    yaws:outh_set_content_length(Sz),
		    accumulate_chunk(FirstChunk),
		    deliver_accumulated(CliSock, GC, SC),
		    stream_loop(CliSock, GC, SC),
		    done;
		_ ->
		    deliver_dyn_file(CliSock, GC, SC, Req, 
				     UT, Bin2,Fd,T,ARG,0)
	    end;
	{data, 0} ->
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin, Fd,T,ARG,N);
	{data, NumChars} ->
	    {Send, Bin2} = skip_data(Bin, Fd, NumChars),
	    accumulate_chunk(Send),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin2, Fd, T,ARG,N);
	{skip, 0} ->
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin, Fd,T,ARG,N);
	{skip, NumChars} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin2, Fd, T,ARG,N);
	{error, NumChars, Str} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    accumulate_chunk(Str),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin2, Fd, T,ARG,N)
    end;

deliver_dyn_file(CliSock, GC, SC, _Req, _UT, _Bin, _Fd, [], _ARG,_N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("deliver_dyn: done~n", []),
    Ret = case yaws:outh_get_chunked() of
	      true ->
		  accumulate_content([crnl(), "0", crnl2()]),
		  continue;
	      false ->
		  done
	  end,
    deliver_accumulated(CliSock, GC, SC),
    Ret.


stream_loop(CliSock, GC, SC) ->
    receive
	{streamcontent, Cont} ->
	    send_streamcontent_chunk(CliSock, GC, SC, Cont),
	    stream_loop(CliSock, GC, SC) ;
	{streamcontent_with_ack, From, Cont} ->	% acknowledge after send
	    send_streamcontent_chunk(CliSock, GC, SC, Cont),
	    From ! {self(), streamcontent_ack},
	    stream_loop(CliSock, GC, SC) ;
	endofstreamcontent ->
	    ok
    after 30000 ->
	    exit(normal)
    end.





send_streamcontent_chunk(CliSock, GC, SC, Chunk) ->
    accumulate_chunk(Chunk),
    Data = erase(acc_content),
    ?Debug("send ~p bytes to ~p ~n", 
	[size(list_to_binary(Data)), CliSock]),
    gen_tcp_send(CliSock, Data, SC, GC).
    

%% what about trailers ??

skip_data(List, Fd, Sz) when list(List) ->
    skip_data(list_to_binary(List), Fd, Sz);
skip_data(Bin, Fd, Sz) when binary(Bin) ->
    ?Debug("Skip data ~p bytes from", [Sz]),
     case  Bin of
	 <<Head:Sz/binary ,Tail/binary>> ->
	     {Head, Tail};
	 _ ->
	     case (catch file:read(Fd, 4000)) of
		 {ok, Bin2} when binary(Bin2) -> 
		     Bin3 = <<Bin/binary, Bin2/binary>>,
		     skip_data(Bin3, Fd, Sz);
		 _Err ->
		     ?Debug("EXIT in skip_data: ~p  ~p ~p~n", [Bin, Sz, _Err]),
		     exit(normal)
	     end
     end;
skip_data({bin, Bin}, _, Sz) ->
    ?Debug("Skip bin data ~p bytes ", [Sz]),
    <<Head:Sz/binary ,Tail/binary>> = Bin,
    {Head, {bin, Tail}};
skip_data({ok, X}, Fd, Sz) ->
    skip_data(X, Fd, Sz).



to_binary(B) when binary(B) ->
    B;
to_binary(L) when list(L) ->
    list_to_binary(L).



accumulate_header(Data) ->
    case get(acc_headers) of
	undefined ->
	    put(acc_headers, [Data, crnl()]);
	List ->
	    put(acc_headers, [List, Data, crnl()])
    end.

accumulate_content(Data) ->
    case get(acc_content) of
	undefined ->
	    put(acc_content, [Data]);
	List ->
	    put(acc_content, [List, Data])
    end.

accumulate_chunk(Data) ->
    case yaws:outh_get_chunked() of
	false ->
	    accumulate_content(Data);
	true ->
	    B = to_binary(Data),
	    case size(B) of
		0 ->
		    skip;
		Len ->
		    CRNL = crnl(),
		    Data2 = [CRNL, yaws:integer_to_hex(Len) , crnl(), B], 
		    accumulate_content(Data2)
	    end
    end.



handle_out_reply(L, LineNo, YawsFile, SC, A) when list (L) ->
    handle_out_reply_l(L, LineNo, YawsFile, SC, A, undefined);

handle_out_reply({html, Html}, _LineNo, _YawsFile, _SC, _A) ->
    accumulate_chunk(Html);

handle_out_reply({ehtml, E}, _LineNo, _YawsFile, SC, A) ->
    case safe_ehtml_expand(E) of
	{ok, Val} ->
	    accumulate_chunk(Val);
	{error, ErrStr} ->
	    handle_crash(A,ErrStr,SC)
    end;

handle_out_reply({content, MimeType, Cont}, _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    accumulate_chunk(Cont);

handle_out_reply({streamcontent, MimeType, First}, 
		 _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    {streamcontent, MimeType, First};

handle_out_reply({streamcontent_with_size, Sz, MimeType, First}, 
		 _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    {streamcontent_with_size, Sz, MimeType, First};

handle_out_reply({header, H},  _LineNo, _YawsFile, _SC, _A) ->
    yaws:accumulate_header(H);

handle_out_reply({allheaders, Hs}, _LineNo, _YawsFile, _SC, _A) ->
    yaws:outh_clear_headers(),
    lists:foreach(fun({header, Head}) -> yaws:accumulate_header(Head) end, Hs);

handle_out_reply({status, Code},_LineNo,_YawsFile,_SC,_A) when integer(Code) ->
    yaws:outh_set_status_code(Code);

handle_out_reply({'EXIT', normal}, _LineNo, _YawsFile, _SC, _A) ->
    exit(normal);

handle_out_reply(break, _LineNo, _YawsFile, _SC, _A) ->
    break;

handle_out_reply({redirect_local, Path0}, _LineNo, _YawsFile, SC, A) ->
    Path = case Path0 of
	       {abs_path, P} ->
		   P;
	       P ->
		   P
	   end,
    Scheme = redirect_scheme(SC),
    Arg = hd(A),
    Headers = Arg#arg.headers,
    HostPort = redirect_host(SC, Headers#headers.host),
    Loc = ["Location: ", Scheme, HostPort, Path, "\r\n"],
    OH = get(outh),
    new_redir_h(OH, Loc),
    ok;



handle_out_reply({redirect, URL}, _LineNo, _YawsFile, _SC, _A) ->
    Loc = ["Location: ", URL, "\r\n"],
    OH = get(outh),
    new_redir_h(OH, Loc),
    ok;

handle_out_reply(ok, _LineNo, _YawsFile, _SC, _A) ->
    ok;

handle_out_reply({'EXIT', Err}, LineNo, YawsFile, SC, ArgL) ->
    A = hd(ArgL),
    L = ?F("~n~nERROR erlang  code  crashed:~n "
	   "File: ~s:~w~n"
	   "Reason: ~p~nReq: ~p~n",
	   [YawsFile, LineNo, Err, A#arg.req]),
    handle_crash(A, L, SC);

handle_out_reply({get_more, Cont, State}, _LineNo, _YawsFile, _SC, _A) ->
    {get_more, Cont, State};

handle_out_reply(Reply, LineNo, YawsFile, SC, ArgL) ->
    A = hd(ArgL),
    L =  ?F("yaws code at ~s:~p crashed or "
	    "ret bad val:~p ~nReq: ~p",
	    [YawsFile, LineNo, Reply, A#arg.req]),
    handle_crash(A, L, SC).

    

handle_out_reply_l([Reply|T], LineNo, YawsFile, SC, A, Res) ->		  
    case handle_out_reply(Reply, LineNo, YawsFile, SC, A) of
	{get_more, Cont, State} ->
	    handle_out_reply_l(T, LineNo, YawsFile, SC, A, {get_more, Cont, State});
	break ->
	    break;
	{page, Page} ->
	    {page, Page};
	_ ->
	    handle_out_reply_l(T, LineNo, YawsFile, SC, A, Res)
    end;
handle_out_reply_l([], _LineNo, _YawsFile, _SC, _A, Res) ->
    Res.



%% Erlang yaws code crashed, display either the
%% actual crash or a custimized error message 
<commit_msg>return last value of the list returned from out/1<commit_after>%%----------------------------------------------------------------------
%%% File    : yaws_server.erl
%%% Author  : <NAME> <<EMAIL>>
%%% Purpose : 
%%% Created : 16 Jan 2002 by <NAME> <<EMAIL>>
%%%----------------------------------------------------------------------

-module(yaws_server).
-author('<EMAIL>').

-compile(export_all).
%%-export([Function/Arity, ...]).

-behaviour(gen_server).

-include_lib("yaws/include/yaws.hrl").
-include_lib("yaws/include/yaws_api.hrl").
-include("yaws_debug.hrl").

-include_lib("kernel/include/file.hrl").

%% External exports
-export([start_link/0]).

%% gen_server callbacks
-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
-import(filename, [join/1]).
-import(lists, [foreach/2, map/2, flatten/1, flatmap/2, reverse/1]).

-import(yaws_api, [ehtml_expand/1]).

-record(gs, {gconf,
	     group,         %% list of #sconf{} s
	     ssl,           %% ssl | nossl
	     l,             %% listen socket
	     mnum = 0,      %% dyn compiled erl module  number
	     sessions = 0,  %% number of HTTP sessions
	     reqs = 0}).    %% number of HTTP requests


%%%----------------------------------------------------------------------
%%% API
%%%----------------------------------------------------------------------
start_link() ->
    gen_server:start_link({local, yaws_server}, yaws_server, [], []).

status() ->
    gen_server:call(?MODULE, status).



stats() -> 
    {S, Time} = status(),
    {_GC, Srvs, _} = S,
    Diff = calendar:time_difference(Time, calendar:local_time()),
    G = fun(L) -> lists:reverse(lists:keysort(2, L)) end,
    
    R= flatmap(
      fun({_Pid, SCS}) ->
	      map(
		fun(SC) ->
			
			E = SC#sconf.ets,
			L = ets:match(E, {{urlc_total, '$1'}, '$2'}),
			{SC#sconf.servername,  
			 flatten(yaws:fmt_ip(SC#sconf.listen)),
			 G(map(fun(P) -> list_to_tuple(P) end, L))}
		end, SCS)
      end, Srvs),
    {Diff, R}.

			      
get_app_args() ->
    AS=init:get_arguments(),
    Debug = case application:get_env(yaws, debug) of
		undefined ->
		    lists:member({yaws, ["debug"]}, AS);
		{ok, Val}  ->
		    Val
	    end,
    Trace = case application:get_env(yaws, trace) of
		undefined ->
		    case {lists:member({yaws, ["trace", "http"]}, AS),
			  lists:member({yaws, ["trace", "traffic"]}, AS)} of
			{true, _} ->
			    {true, http};
			{_, true} ->
			    {true, traffic};
			_ ->
			    false
		    end;
		{ok, http} ->
		    {true, http};
		{ok, traffic} ->
		    {true, traffic};
		_ ->
		    false
	    end,
    Conf = case application:get_env(yaws, conf) of
	       undefined ->
		   find_c(AS);
	       {ok, File} ->
		   {file, File}
	   end,
    RunMod = case application:get_env(yaws, runmod) of
		 undefined ->
		     find_runmod(AS);
		 {ok,Mod} ->
		     {ok,Mod}
	     end,
    Embed = case application:get_env(yaws, embedded) of
		undefined ->
		    false;
		{ok, Val0} ->
		    Val0
	    end,
    {Debug, Trace, Conf, RunMod, Embed}.

find_c([{conf, [File]} |_]) ->
    {file, File};
find_c([_|T]) ->
    find_c(T);
find_c([]) ->
    false.

find_runmod([{runmod, [Mod]} |_]) ->
    {ok,l2a(Mod)};
find_runmod([_|T]) ->
    find_runmod(T);
find_runmod([]) ->
    false.

l2a(L) when list(L) -> list_to_atom(L);
l2a(A) when atom(A) -> A.



%%----------------------------------------------------------------------
%% Func: init/1
%% Returns: {ok, State}          |
%%          {ok, State, Timeout} |
%%          ignore               |
%%          {stop, Reason}
%%----------------------------------------------------------------------
init([]) ->
    process_flag(trap_exit, true),
    put(start_time, calendar:local_time()),  %% for uptime
    {Debug, Trace, Conf, RunMod, Embed} = get_app_args(),
    case Embed of 
	false ->
	    case yaws_config:load(Conf, Trace, Debug) of
		{ok, Gconf, Sconfs} ->
		    erase(logdir),
		    ?Debug("Conf = ~p~n", [?format_record(Gconf, gconf)]),
		    yaws_log:setdir(Gconf#gconf.logdir, Sconfs),
		    case Gconf#gconf.trace of
			{true, What} ->
			    yaws_log:open_trace(What);
			_ ->
			    ok
		    end,
		    init2(Gconf, Sconfs, RunMod, true);
		{error, E} ->
		    case erase(logdir) of
			undefined ->
			    error_logger:error_msg("Yaws: Bad conf: ~p~n", 
						   [E]),
			    init:stop(),
			    {stop, E};
			Dir ->
			    yaws_log:setdir(Dir, []),
			    error_logger:error_msg("Yaws: bad conf: ~s~n",[E]),
			    init:stop(),
			    {stop, E}
		    end
	    end;
	true ->
	    init2(yaws_config:make_default_gconf(Debug), [], undef, true)
    end.


init2(Gconf, Sconfs, RunMod, FirstTime) ->

    lists:foreach(
      fun(D) ->
	      yaws_debug:format(Gconf, "Add path ~p~n", [D]),
	      code:add_pathz(D)
      end, Gconf#gconf.ebin_dir),


    %% start user provided application
    case RunMod of
	{ok,Mod} -> 
	    runmod(Mod);
	_ -> 
	    false
    end,
    lists:foreach(fun(M) -> runmod(M) end, Gconf#gconf.runmods),

    %% start the individual server processes
    L = lists:map(
	  fun(Group) ->
		  proc_lib:start_link(?MODULE, gserv, [Gconf, Group])
	  end, Sconfs),
    L2 = lists:zf(fun({error, F, A}) ->	
			  error_logger:error_msg(F, A),
			  exit(nostart);
		     ({error, Reason}) ->
			  error_logger:error_msg("FATAL: ~p~n", [Reason]),
			  exit(nostart);
		     ({_Pid, _SCs}) ->
			  true;
		     (none) ->
			  false
		  end, L),
    ?Debug("L=~p~n", [yaws_debug:nobin(L)]),
    if
	FirstTime == true ->
	    CTL = proc_lib:spawn_link(yaws_ctl, start, 
				      [self(), Gconf#gconf.uid]),
	    receive
		{CTL, ok} ->
		    ok;
		{CTL, {error, R}} ->
		    exit(R)
	    end;
		
	true ->
	    ok
    end,
    file:make_dir("/tmp/yaws"),
    set_writeable("/tmp/yaws"),
    case (catch yaws_log:uid_change(Gconf)) of
	ok ->
	    ok;
	ERR ->
	    error_logger:error_msg("Failed to change uid on logfiles:~n~p",
				   [ERR]),
	    exit(error)
    end,

    %% and now finally, we've opened the ctl socket and are
    %% listening to all sockets we can possibly change username
 

    GC2 = case (catch yaws:setuser(Gconf#gconf.username)) of
	      ignore ->
		  Gconf;
	      {ok, NewUid} ->
		  error_logger:info_msg("Changed uid to ~s~n", [NewUid]),
		  Gconf#gconf{uid = NewUid};
	      Other ->
		  error_logger:error_msg("Failed to set user:~n~p", [Other]),
		  exit(Other)
	  end,
    case (catch setup_tdir(GC2)) of
	ok ->
	    ok;
	_Error ->
	    exit(error)
    end,
    lists:foreach(fun({Pid, _}) -> Pid ! {newuid, GC2#gconf.uid} end, L2),
    {ok, {GC2, L2, 0}}.


%%----------------------------------------------------------------------
%% Func: handle_call/3
%% Returns: {reply, Reply, State}          |
%%          {reply, Reply, State, Timeout} |
%%          {noreply, State}               |
%%          {noreply, State, Timeout}      |
%%          {stop, Reason, Reply, State}   | (terminate/2 is called)
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_call(status, _From, State) ->
    Reply = {State, get(start_time)},
    {reply, Reply, State};
handle_call(pids, _From, State) ->  %% for gprof
    L = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    {reply, [self() | L], State};

handle_call(mnum, _From, {GC, Group, Mnum}) ->
    {reply, Mnum+1,   {GC, Group, Mnum+1}};

handle_call({setconf, GC, Groups}, _From, State) ->
    %% First off, terminate all currently running processes
    Curr = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    lists:foreach(fun(Pid) ->
			  Pid ! {self(), stop},
			  receive
			      {Pid, ok} ->
				  ok
			  end
		  end, Curr),
    case init2(GC, Groups, undef, false) of
	{ok, State2} ->
	    {reply, ok, State2};
	Err ->
	    {reply, Err, {GC, [], 0}}
    end;

handle_call(getconf, _From, State) ->
    {GC, Pairs, _} = State,
    Groups = lists:map(fun({_Pid, SCs}) -> SCs end, Pairs),
    {reply, {ok, GC, Groups}, State}.
			       
			      



%%----------------------------------------------------------------------
%% Func: handle_cast/2
%% Returns: {noreply, State}          |
%%          {noreply, State, Timeout} |
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_cast(_Msg, State) ->
    {noreply, State}.

%%----------------------------------------------------------------------
%% Func: handle_info/2
%% Returns: {noreply, State}          |
%%          {noreply, State, Timeout} |
%%          {stop, Reason, State}            (terminate/2 is called)
%%----------------------------------------------------------------------
handle_info({'EXIT', Pid, Reason},  State) ->
    L = lists:map(fun(X) ->element(1, X) end, element(2, State)),
    case lists:member(Pid, L) of
	true ->
	    %% one of our gservs died
	    error_logger:format("yaws: FATAL gserv died ~p~n", [Reason]),
	    exit(restartme);
	false ->
	    ignore
    end,
    {noreply, State};


handle_info(_Msg, State) ->
    ?Debug("GOT ~p~n", [_Msg]),
    {noreply, State}.

%%----------------------------------------------------------------------
%% Func: terminate/2
%% Purpose: Shutdown the server
%% Returns: any (ignored by gen_server)
%%----------------------------------------------------------------------
terminate(_Reason, _State) ->
    ok.


%% we search and setup www authenticate for each directory
%% specified as an auth directory. These are merged with server conf.

setup_auth(SC) ->
    ?Debug("setup_auth(~p)", [yaws_debug:nobin(SC)]),
    ?f(lists:map(fun(Auth) ->
		      add_yaws_auth(Auth#auth.dir, Auth)
	      end, SC#sconf.authdirs)).

add_yaws_auth(Dirs, A) ->
    lists:map(
      fun(Dir) ->
	  case file:consult([Dir, [$/|".yaws_auth"]]) of
	      {ok, [{realm, Realm} |TermList]} ->
		  {Dir, A#auth{realm = Realm,
			       users = TermList++A#auth.users}};
	      {ok, TermList} ->
		  {Dir, A#auth{users = TermList++A#auth.users}};
	      {error, enoent} ->
		  {Dir, A};
	      _Err ->
		  error_logger:format("Bad .yaws_auth file in dir ~p~n", [Dir]),
		  {Dir, A}
	  end
      end, Dirs).


do_listen(SC) ->
    case SC#sconf.ssl of
	undefined ->
	    {nossl, gen_tcp:listen(SC#sconf.port, opts(SC))};
	SSL ->
	    {ssl, ssl:listen(SC#sconf.port, ssl_opts(SC, SSL))}
    end.

set_writeable(Dir) ->
    set_dir_mode(Dir, 8#777).

set_dir_mode(Dir, Mode) ->
    {ok, FI} = file:read_file_info(Dir),
    file:write_file_info(Dir, FI#file_info{mode = Mode}).


gserv(_, []) ->
    proc_lib:init_ack(none);

%% One server per IP we listen to
gserv(GC, Group0) ->
    process_flag(trap_exit, true),
    ?TC([{record, GC, gconf}]),
    Group = map(fun(SC) -> 
			E = ets:new(yaws_code, [public, set]),
			ets:insert(E, {num_files, 0}),
			ets:insert(E, {num_bytes, 0}),
			Auth = setup_auth(SC),
			SC#sconf{ets = E,
				 authdirs = Auth}
		end, Group0),
    SC = hd(Group),
    case do_listen(SC) of
	{SSLBOOL, {ok, Listen}} ->
	    call_start_mod(SC),
	    error_logger:info_msg(
	      "Yaws: Listening to ~s:~w for servers~s~n",
	      [yaws:fmt_ip(SC#sconf.listen),
	       SC#sconf.port,
	       catch map(fun(S) ->  
				 io_lib:format("~n - ~s under ~s",
					       [S#sconf.servername,
						S#sconf.docroot])
			 end, Group)
	      ]),
	    proc_lib:init_ack({self(), Group}),
	    N = receive
		    {newuid, UID} ->
			UID
		end,
	    GS = #gs{gconf = GC#gconf{uid = N},
		     group = Group,
		     ssl = SSLBOOL,
		     l = Listen},
	    acceptor(GS),
	    gserv(GS, [], 0);
	{_,Err} ->
	    error_logger:format("Yaws: Failed to listen ~s:~w  : ~p~n",
				[yaws:fmt_ip(SC#sconf.listen),
				 SC#sconf.port, Err]),
	    proc_lib:init_ack({error, "Can't listen to socket: ~p ",[Err]}),
	    exit(normal)
    end.
			

setup_tdir(GC) ->
    Tdir = "/tmp/yaws/" ++ GC#gconf.uid,
    file:make_dir(Tdir),
    set_dir_mode(Tdir, 8#700),
    case file:list_dir(Tdir) of
	{ok, Files0} ->
	    case lists:zf(
		   fun(F) -> 
			   Fname = Tdir ++ "/" ++ F,
			   case file:delete(Fname) of
			       ok ->
				   false;
			       {error, Reason} ->
				   {true, {Fname, Reason}}
			   end
		   end, Files0) of
		[] ->
		    ok;
		[{FF, RR}|_] ->
		    error_logger:format("Failed to delete ~p~n~p", [FF, RR]),
		    {error, RR}
	    end;
	{error, Reason} ->
	    error_logger:format("Failed to list ~p probably "
				"due to permission errs: ~p",
				[Tdir, Reason]),
	    {error, Reason}
    end.




gserv(GS, Ready, Rnum) ->
    receive
	{From , status} ->
	    From ! {self(), GS},
	    gserv(GS, Ready, Rnum);
	{From, next} when Ready == [] ->
	    acceptor(GS),
	    gserv(GS, Ready, Rnum);
	{From, next} ->
	    [R|RS] = Ready,
	    R ! {self(), accept},
	    gserv(GS, RS, Rnum-1);
	{From, done_client, Int} ->
	    GS2 = GS#gs{sessions = GS#gs.sessions + 1,
			reqs = GS#gs.reqs + Int},
	    if
		Rnum == 8 ->
		    From ! {self(), stop},
		    gserv(GS2, Ready, Rnum);
		Rnum < 8 ->
		    gserv(GS2, [From | Ready], Rnum+1)
	    end;
	{'EXIT', _Pid, _} ->
	    gserv(GS, Ready, Rnum);
	{From, stop} ->   
	    unlink(From),
	    {links, Ls} = process_info(self(), links),
	    lists:foreach(fun(X) -> unlink(X), exit(X, kill) end, Ls),
	    From ! {self(), ok},
	    exit(normal)
    end.
	    
 
call_start_mod(SC) ->
    case SC#sconf.start_mod of
	undefined ->
	    ok;
	Mod0 ->
	    Mod = l2a(Mod0),
	    case code:ensure_loaded(Mod) of
		{module, Mod} ->
		    spawn(Mod, start, [SC]);
		Err ->
		    error_logger:format("Cannot load module ~p: ~p~n", 
					[Mod,Err])
	    end
    end.



opts(SC) ->
    [binary, 
     {ip, SC#sconf.listen},
     {packet, http},
     {recbuf, 8192},
     {reuseaddr, true},
     {active, false}
    ].



ssl_opts(SC, SSL) ->
    Opts = [
	    binary,
	    {ip, SC#sconf.listen},
	    {packet, 0},
	    {active, false} | ssl_opts(SSL)],
    ?Debug("SSL opts  ~p~n", [Opts]),
    Opts.




ssl_opts(SSL) ->
    L = [if SSL#ssl.keyfile /= undefined ->
		 {keyfile, SSL#ssl.keyfile};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.certfile /= undefined ->
		 {certfile, SSL#ssl.certfile};
	    true ->
		 false
	 end,

	 if SSL#ssl.cacertfile /= undefined ->
		 {cacertfile, SSL#ssl.cacertfile};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.verify /= undefined ->
		 {verify, SSL#ssl.verify};
	    true ->
		 false
	 end,
	 
	 if SSL#ssl.password /= undefined ->
		 {password, <PASSWORD>};
	    true ->
		 false
	 end,
	 if SSL#ssl.ciphers /= undefined ->
		 {ciphers, SSL#ssl.ciphers};
	    true ->
		 false
	 end
	],
    filter_false(L).


filter_false(L) ->
    [X || X <- L, X /= false].

	   
	 
do_accept(GS) when GS#gs.ssl == nossl ->
    gen_tcp:accept(GS#gs.l);
do_accept(GS) when GS#gs.ssl == ssl ->
    ssl:accept(GS#gs.l).


acceptor(GS) ->
    proc_lib:spawn_link(yaws_server, acceptor0, [GS, self()]).
acceptor0(GS, Top) ->
    ?TC([{record, GS, gs}]),
    X = do_accept(GS),
    ?Debug("do_accept ret: ~p~n", [X]),
    Top ! {self(), next},
    case X of
	{ok, Client} ->
	    case (GS#gs.gconf)#gconf.trace of  %% traffic trace
		{true, _} ->
		    {ok, {IP, Port}} = case GS#gs.ssl of
					   ssl ->
					       ssl:peername(Client);
					   nossl ->
					       inet:peername(Client)
				       end,
		    Str = ?F("New (~p) connection from ~s:~w~n", 
			     [GS#gs.ssl, yaws:fmt_ip(IP),Port]),
		    yaws_log:trace_traffic(from_client, Str);
		_ ->
		    ok
	    end,

	    Res = (catch aloop(Client, GS,  0)),
	    ?Debug("RES = ~p~n", [Res]),
	    if
		GS#gs.ssl == nossl ->
		    gen_tcp:close(Client);
		GS#gs.ssl == ssl ->
		    ssl:close(Client)
	    end,
	    case Res of
		{ok, Int} when integer(Int) ->
		    Top ! {self(), done_client, Int};
		{'EXIT', normal} ->
		    exit(normal);
		{'EXIT', {badarg, [{erlang, apply, _} |_]}} ->
		    %% hopeless cracker request
		    %% don't crash log
		    exit(normal);
		{'EXIT', Reason} ->
		    error_logger:error_msg("Yaws process died: ~p~n", [Reason]),
		    exit(normal)
	    end,
	    %% we cache processes
	    receive
		{Top, stop} ->
		    exit(normal);
		{Top, accept} ->
		    acceptor0(GS, Top)
	    end;
	_ ->
	    exit(normal)
    end.
     


%%%----------------------------------------------------------------------
%%% Internal functions
%%%----------------------------------------------------------------------

aloop(CliSock, GS, Num) when GS#gs.ssl == nossl ->
    ?TC([{record, GS, gs}]),
    case http_get_headers(CliSock, GS#gs.gconf) of
	done ->
	    {ok, Num};
	{Req, H} ->
	    SC = pick_sconf(GS#gs.gconf, H, GS#gs.group),
	    ?Debug("SC: ~p", [?format_record(SC, sconf)]),
	    ?TC([{record, SC, sconf}]),
	    ?Debug("Headers = ~p~n", [?format_record(H, headers)]),
	    ?Debug("Request = ~p~n", [?format_record(Req, http_request)]),
	    IP = case SC#sconf.access_log of
		     true ->
			 {ok, {Ip, _Port}} = inet:peername(CliSock),
			 Ip;
		     _ ->
			 undefined
		 end,
	    put(outh, #outh{}),
	    Res = apply(yaws_server, Req#http_request.method, 
			[CliSock, GS#gs.gconf, SC, Req, H]),
	    maybe_access_log(IP, SC, Req, H),
	    case Res of
		continue ->
		    aloop(CliSock, GS, Num+1);
		done ->
		    {ok, Num+1};
		{page, Page} ->
		    put(outh, #outh{}),
		    apply(yaws_server, Req#http_request.method, 
			  [CliSock, GS#gs.gconf, SC#sconf{appmods=[]}, 
			   Req#http_request{path = {abs_path, Page}}, H])
	    end
    end;



aloop(CliSock, GS, Num) when GS#gs.ssl == ssl ->
    ?TC([{record, GS, gs}]),
    case yaws_ssl:ssl_get_headers(CliSock, GS#gs.gconf) of
	done ->
	    {ok, Num};
	{ok, Req, H, Trail} ->
	    put(ssltrail, Trail),  %% hack hack hack
	    SC = hd(GS#gs.group),
	    IP = case SC#sconf.access_log of
		     true ->
			 {ok, {Ip, _Port}} = ssl:peername(CliSock),
			 Ip;
		     _ ->
			 undefined
		 end,
	    put(outh, #outh{}),
	    Res = apply(yaws_server, Req#http_request.method, 
			[CliSock, GS#gs.gconf, SC, Req, H]),
	    maybe_access_log(IP, SC, Req, H),
	    case Res of
		continue ->
		    aloop(CliSock, GS, Num+1);
		done ->
		    {ok, Num+1};
		{page, Page} ->
		    put(outh, #outh{}),
		    apply(yaws_server, Req#http_request.method, 
			  [CliSock, GS#gs.gconf, SC#sconf{appmods = []}, 
			   Req#http_request{path = {abs_path, Page}}, H])
	    end
    end.



pick_sconf(GS, H, Group) ->
    case H#headers.host of
	undefined ->
	    hd(Group);
	Host ->
	    pick_host(GS, Host, Group, Group)
    end.


pick_host(_GC, Host, [H|_T], _Group) when H#sconf.servername == Host ->
    H;
pick_host(GC, Host, [_|T], Group) ->
    pick_host(GC, Host, T, Group);
pick_host(_GC, _Host, [], Group) ->
    hd(Group).


inet_peername(Sock, SC) ->
    case SC#sconf.ssl of
	undefined ->
	    inet:peername(Sock);
	_SSL ->
	    ssl:peername(Sock)
    end.
	    

maybe_access_log(Ip, SC, Req, H) ->
    ?TC([{record, SC, sconf}]),
    Status = case yaws:outh_get_status_code() of
		 undefined -> "-";
		 I -> integer_to_list(I)
	     end,
    Len = case yaws:outh_get_contlen() of
	      undefined ->
		  "-";
	      I2 -> integer_to_list(I2)
	  end,
    Ver = case Req#http_request.version of
	      {1,0} ->
		  "HTTP/1.0";
	      {1,1} ->
		  "HTTP/1.1";
	      {0,9} ->
		  "HTTP/0.9" 
	  end,
    case SC#sconf.access_log of
	true ->
	    Path = safe_decode_path(Req#http_request.path),
	    Meth = atom_to_list(Req#http_request.method),
	    Referrer = optional_header(H#headers.referer),
	    UserAgent = optional_header(H#headers.user_agent),
	    yaws_log:accesslog(SC#sconf.servername, Ip, 
			       [Meth, $\s, Path, $\s, Ver], 
			       Status, Len, Referrer, UserAgent);
	false ->
	    ignore
    end.

safe_decode_path(Path) ->
    case (catch decode_path(Path)) of
	{'EXIT', _} ->
	    "/undecodable_path";
	Val ->
	    Val 
    end.


decode_path({abs_path, Path}) ->
    yaws_api:url_decode(Path).

optional_header(Item) ->
    case Item of
	undefined -> "-";
	Item -> Item
    end.

do_recv(Sock, Num, TO, nossl) ->
    gen_tcp:recv(Sock, Num, TO);
do_recv(Sock, Num, _TO, ssl) ->
    case erase(ssltrail) of %% hack from above ...
	undefined ->
	    split_recv(ssl:recv(Sock, 0), Num);   %% ignore Num val ??? TO ??
	Bin ->
	    {ok, Bin}
    end.

%% weird ... ssl module doesn't respect Num arg for binary socks
split_recv({ok, B}, Num) when integer(Num) ->
    case B of
	<<Bin:Num/binary , Tail/binary>> ->
	    put(ssltrail, Tail),
	    {ok, Bin};
	_ ->
	    {ok, B}
    end;
split_recv(E, _) ->
    E.


	
	 

cli_recv(S, Num, GC, SslBool) when GC#gconf.trace == false ->
    do_recv(S, Num, GC#gconf.timeout, SslBool);
cli_recv(S, Num, GC, SslBool) ->
    Res = do_recv(S, Num, GC#gconf.timeout, SslBool),
    case Res of
	{ok, Val} when tuple(Val) ->
	    yaws_log:trace_traffic(from_client, ?F("~p~n", [Val]));
	{error, What} ->
	    yaws_log:trace_traffic(from_client, ?F("~n~p~n", [What]));
	{ok, http_eoh} ->
	    ok;
	{ok, Val} when GC#gconf.trace == {true, traffic} ->
	    yaws_log:trace_traffic(from_client, Val);
	{ok, Val} ->
	    {ok, Val}
    end,
    Res.



gen_tcp_send(S, Data, SC, GC) ->
    Res = case SC#sconf.ssl of
	      undefined ->
		  gen_tcp:send(S, Data);
	      _SSL ->
		  ssl:send(S, Data)
	  end,
    case GC#gconf.debug of
	false ->
	    case Res of
		ok ->
		    ok;
		Err ->
		    exit(Err)
	    end;
	true ->
	    case Res of
		ok ->
		    ?Debug("Sent ~p~n", [yaws_debug:nobin(Data)]),
		    ok;
		Err ->
		    {B2, Size} = strip(Data),
		    yaws_debug:derror(GC, "Failed to send ~w bytes:~n~p "
				      "on socket ~p: ~p~n",
				      [Size, B2, S, Err]),
		    exit(Err)
	    end
    end.


strip(Data) ->
    L = list_to_binary([Data]),
    case L of
	<<Head:50/binary, _/binary>> ->
	    {binary_to_list(<<Head/binary, ".....">>), size(L)};
	_ ->
	    {binary_to_list(L), size(L)}
    end.
	 


http_get_headers(CliSock, GC) ->
    ?TC([{record, GC, gconf}]),
    inet:setopts(CliSock, [{packet, http}]),
    R = http_recv_request(CliSock, GC),
    H = http_get_headers(CliSock, R, GC, #headers{}),
    {R, H}.


http_recv_request(CliSock, GC) ->
    case cli_recv(CliSock, 0, GC, nossl) of
	{ok, R} when element(1, R) == http_request ->
	    R;
	{error, {http_error, "\r\n"}} ->
	    http_recv_request(CliSock, GC);
	{error, {http_error, "\n"}} ->
	    http_recv_request(CliSock, GC);
	_Other ->
	    ?Debug("Got ~p~n", [_Other]),
	    exit(normal)
    end.


		  
http_get_headers(CliSock, Req, GC, H) ->
    ?TC([{record, GC, gconf}]),
    case cli_recv(CliSock, 0, GC, nossl) of
	{ok, {http_header,  _Num, 'Host', _, Host}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{host = Host});
	{ok, {http_header, _Num, 'Connection', _, Conn}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{connection = Conn});
	{ok, {http_header, _Num, 'Accept', _, Accept}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{accept = Accept});
	{ok, {http_header, _Num, 'If-Modified-Since', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			     H#headers{if_modified_since = X});
	{ok, {http_header, _Num, 'If-Match', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_match = X});
	{ok, {http_header, _Num, 'If-None-Match', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_none_match = X});
	{ok, {http_header, _Num, 'If-Range', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{if_range = X});
	{ok, {http_header, _Num, 'If-Unmodified-Since', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{if_unmodified_since = X});
	{ok, {http_header, _Num, 'Range', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{range = X});
	{ok, {http_header, _Num, 'Referer',_, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{referer = X});
	{ok, {http_header, _Num, 'User-Agent', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{user_agent = X});
	{ok, {http_header, _Num, 'Accept-Ranges', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{accept_ranges = X});
	{ok, {http_header, _Num, 'Cookie', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{cookie = [X|H#headers.cookie]});
	{ok, {http_header, _Num, 'Keep-Alive', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{keep_alive = X});
	{ok, {http_header, _Num, 'Content-Length', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{content_length = X});
	{ok, {http_header, _Num, 'Content-Type', _, X}} ->
	    http_get_headers(CliSock, Req, GC, H#headers{content_type = X});
	{ok, {http_header, _Num, 'Authorization', _, X}} ->
	    http_get_headers(CliSock, Req, GC, 
			H#headers{authorization = parse_auth(X)});
	{ok, http_eoh} ->
	    H;
	{error, {http_error, "\r\n"}} ->
	     http_get_headers(CliSock, Req, GC, H);
	{error, {http_error, "\n"}} ->
	     http_get_headers(CliSock, Req, GC, H);
	{ok, X} ->
	    ?Debug("OTHER header ~p~n", [X]),
	    http_get_headers(CliSock, Req, GC, 
			     H#headers{other=[X|H#headers.other]});
	_Err ->
	    exit(normal)
    
    end.


inet_setopts(SC, S, Opts) when SC#sconf.ssl == undefined ->
    inet:setopts(S, Opts);
inet_setopts(_,_,_) ->
    ok.  %% noop for ssl


%% ret:  continue | done
'GET'(CliSock, GC, SC, Req, Head) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("GET ~p", [?format_record(Req, http_request)]),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    flush(SC, CliSock, Head#headers.content_length),
    ARG = make_arg(CliSock, Head, Req, GC, SC),
    ARG2 = apply(SC#sconf.arg_rewrite_mod, arg_rewrite, [ARG]),
    handle_request(CliSock, GC, SC, ARG2, 0).


'POST'(CliSock, GC, SC, Req, Head) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("POST Req=~p H=~p~n", [?format_record(Req, http_request),
				  ?format_record(Head, headers)]),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    PPS = SC#sconf.partial_post_size,
    Bin = case Head#headers.content_length of
	      undefined ->
		  case Head#headers.connection of
		      "close" ->
			  get_client_data(CliSock, all, GC,
					  is_ssl(SC#sconf.ssl));
		      _ ->
			  ?Debug("No content length header ",[]),
			  exit(normal)
		  end;
	      Len when integer(PPS) ->
		  Int_len = list_to_integer(Len),
		  if 
		      Int_len == 0 ->
			  <<>>;
		      PPS < Int_len ->
			  {partial, get_client_data(CliSock, PPS, GC,
						    is_ssl(SC#sconf.ssl))};
		      true ->
			  get_client_data(CliSock, Int_len, GC, 
					  is_ssl(SC#sconf.ssl))
		  end;
	      Len when PPS == nolimit ->
		  Int_len = list_to_integer(Len),
		  if
		      Int_len == 0 ->
			  <<>>;
		      true ->
			  get_client_data(CliSock, Int_len, GC, 
					  is_ssl(SC#sconf.ssl))
		  end
	  end,
    ?Debug("POST data = ~s~n", [binary_to_list(un_partial(Bin))]),
    ARG = make_arg(CliSock, Head, Req, GC, SC),
    ARG2 = ARG#arg{clidata = Bin},
    ARG3 = apply(SC#sconf.arg_rewrite_mod, arg_rewrite, [ARG2]),
    handle_request(CliSock, GC, SC, ARG3, size(un_partial(Bin))).


is_ssl(undefined) ->
    nossl;
is_ssl(R) when record(R, ssl) ->
    ssl.

un_partial({partial, Bin}) ->
    Bin;
un_partial(Bin) ->
    Bin.

%% will throw
'HEAD'(CliSock, GC, SC, Req, Head) ->
    'GET'(CliSock, GC, SC, Req, Head).

'TRACE'(_CliSock, _GC, _SC, _Req, _Head) ->
    nyi.

'OPTIONS'(CliSock, GC, SC, Req, Head) ->
    ?Debug("OPTIONS", []),
    ok = inet_setopts(SC, CliSock, [{packet, raw}, binary]),
    flush(SC, CliSock, Head#headers.content_length),
    _ARG = make_arg(CliSock, Head, Req, GC, SC),
    ?Debug("OPTIONS delivering", []),
    deliver_options(CliSock, Req, GC, SC).

make_arg(CliSock, Head, Req, _GC, SC) ->
    ?TC([{record, _GC, gconf}, {record, SC, sconf}]),
    #arg{clisock = CliSock,
	 headers = Head,
	 req = Req,
	 opaque = SC#sconf.opaque,
	 pid = self(),
	 docroot = SC#sconf.docroot}.

handle_request(CliSock, GC, SC, ARG, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    Req = ARG#arg.req,
    H = ARG#arg.headers,
    {abs_path, RawPath} = Req#http_request.path,
    case (catch yaws_api:url_decode_q_split(RawPath)) of
	{'EXIT', _} ->   %% weird broken cracker requests
	    deliver_403(CliSock, Req, GC, SC);
	{DecPath, QueryPart} ->
	    UT =  url_type(GC, SC, DecPath, QueryPart),
	    ARG2 = ARG#arg{fullpath=UT#urltype.fullpath},
	    case SC#sconf.authdirs of
		[] ->
		    handle_ut(CliSock, GC, SC, Req, H, ARG2, UT, N);
		_Adirs ->
		    %% we have authentication enabled, check auth
		    UT2 = unflat(UT),
		    case is_authenticated(SC, UT2, Req, H) of
			true ->
			    handle_ut(CliSock, GC, SC, Req, H, ARG2, UT2, N);
			{false, Realm} ->
			    deliver_401(CliSock, Req, GC, Realm, SC)
		    end
	    end
    end.


unflat(U) ->
    U2 = case U#urltype.path of
	     {noflat, F} ->
		 U#urltype{path = lists:flatten(F)};
	     _ ->
		 U
	 end,
    U2.

is_authdir(Req_dir, [{Auth_dir, Auth}|T]) ->
    case lists:prefix(Auth_dir, Req_dir) of
	true ->
	    {true, Auth};
	false ->
	    is_authdir(Req_dir, T)
    end;
is_authdir(_, []) ->
    false.

is_authenticated(SC, UT, _Req, H) ->
    case is_authdir(UT#urltype.path, SC#sconf.authdirs) of
	{true, #auth{dir = _Dir, realm = Realm, users = Users}} ->
	    case H#headers.authorization of
		undefined ->
		    {false, Realm};
		{User, Password} ->
		    case lists:member({User, Password}, Users) of
			true ->
			    true;
			false ->
			    {false, "Got username"}
		    end
	    end;
	false ->
	    true
    end.


handle_ut(CliSock, GC, SC, Req, H, ARG, UT, N) ->
    case UT#urltype.type of
	error ->
	    A2 = ARG#arg{querydata = UT#urltype.q},
	    yaws:outh_set_dyn_headers(Req, H),
	    do_appmod(SC#sconf.errormod_404, out404, CliSock, GC, SC, 
		      Req, [A2, GC, SC], UT, N);
	directory when SC#sconf.dir_listings == true ->
	    P = UT#urltype.dir,
	    yaws:outh_set_dyn_headers(Req, H),
	    yaws_ls:list_directory(CliSock, UT#urltype.data, P, Req, GC, SC);
	directory ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_403(CliSock, Req, GC, SC);
	regular ->
	    ETag = yaws:make_etag(UT#urltype.finfo),
	    case H#headers.if_none_match of
		undefined ->
		    yaws:outh_set_static_headers(Req, UT, H),
		    deliver_file(CliSock, GC, SC, Req, UT);
		ETag ->
		    yaws:outh_set_304_headers(Req, UT, H),
		    deliver_accumulated(CliSock, GC, SC),
		    case yaws:outh_get_doclose() of
			true  -> done;
			false -> continue
		    end;
		_ ->
		    yaws:outh_set_static_headers(Req, UT, H),
		    deliver_file(CliSock, GC, SC, Req, UT)
	    end;
	yaws ->
	    yaws:outh_set_dyn_headers(Req, H),
	    do_yaws(CliSock, GC, SC, Req,  
		    [ARG#arg{querydata = UT#urltype.q}], UT, N);
	forbidden ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_403(CliSock, Req, GC, SC);
	redir_dir ->
	    yaws:outh_set_dyn_headers(Req, H),
	    deliver_302(CliSock, Req, GC, SC, ARG);
	appmod ->
	    yaws:outh_set_dyn_headers(Req, H),
	    close_if_HEAD(Req, 
			  fun() ->
				  deliver_accumulated(CliSock, GC,SC),
				  throw({ok, 1}) 
			  end),
	    {Mod, PathData} = UT#urltype.data,
	    A2 = ARG#arg{appmoddata = PathData,
			 querydata = UT#urltype.q,
			 appmod_prepath = UT#urltype.path},
	    do_appmod(Mod, out, CliSock, GC, SC, Req,  [A2], UT, N)
    end.

	
parse_auth("Basic " ++ Auth64) ->
    case decode_base64(Auth64) of
	{error, _Err} ->
	    undefined;
	Auth ->
	    case string:tokens(Auth, ":") of
		[User, Pass] ->
		    {User, Pass};
		_ ->
		    undefined
	    end
    end;
parse_auth(_) ->
    undefined.




%% we may have content, 

new_redir_h(OH, Loc) ->
    OH2 = OH#outh{status = 302,
		  location = Loc},
    put(outh, OH2).



% we must deliver a 302 if the browser asks for a dir
% without a trailing / in the HTTP req
% otherwise the relative urls in /dir/index.html will be broken.


deliver_302(CliSock, Req, GC, SC, Arg) ->
    ?Debug("in redir 302 ",[]),
    H = get(outh),

    Scheme = redirect_scheme(SC),
    Headers = Arg#arg.headers,
    DecPath = decode_path(Req#http_request.path),
    Loc = case string:tokens(DecPath, "?") of
	      [P] ->
		  ["Location: ", Scheme,
		   Headers#headers.host, P, "/\r\n"];
	      [P, Q] ->
		  ["Location: ", Scheme,
		   Headers#headers.host, P, "/", Q, "\r\n"]
	  end,
    
    Ret = case yaws:outh_get_chunked() of
	      true ->
		  accumulate_content([crnl(), "0", crnl2()]),
		  continue;
	      false ->
		  done
	  end,

    new_redir_h(H, Loc),
    deliver_accumulated(CliSock, GC, SC),
    Ret.


redirect_scheme(SC) ->
    case {SC#sconf.ssl,SC#sconf.rmethod} of
	{_, Method} when list(Method) ->
	    Method++"://";
	{undefined,_} ->
	    "http://";
	{_SSl,_} ->
	    "https://"
    end.    

redirect_host(SC, HostHdr) ->
    case SC#sconf.rhost of
	undefined ->
	    HostHdr;
	_ ->
	    SC#sconf.rhost
    end.

redirect_port(SC) ->
    case {SC#sconf.ssl, SC#sconf.port} of
	{undefined, 80} ->
               "";
	{undefined, Port} -> 
               [$:|integer_to_list(Port)];
	{_SSL, 443} ->
               "";
	{_SSL, Port} -> 
               [$:|integer_to_list(Port)]
    end.    

redirect_scheme_port(SC) ->
    Scheme = redirect_scheme(SC),
    PortPart = redirect_port(SC),
    {Scheme, PortPart}.

servername_sans_port(Servername) ->
    case string:chr(Servername, $:) of
	0 ->
	    Servername;
	N ->
	    lists:sublist(Servername, N-1)
    end.

deliver_options(CliSock, _Req, GC, SC) ->
    H = #outh{status = 200,
	      doclose = false,
	      chunked = false,
	      server = yaws:make_server_header(),
	      allow = yaws:make_allow_header()},
    put(outh, H),
    deliver_accumulated(CliSock, GC, SC),
    continue.

deliver_401(CliSock, _Req, GC, Realm, SC) ->
    B = list_to_binary("<html> <h1> 401 authentication needed  "
		       "</h1></html>"),
    H = #outh{status = 401,
	      doclose = true,
	      chunked = false,
	      server = yaws:make_server_header(),
	      connection = yaws:make_connection_close_header(true),
	      content_length = yaws:make_content_length_header(size(B)),
	      www_authenticate = yaws:make_www_authenticate_header(Realm),
	      contlen = size(B),
	      content_type = yaws:make_content_type_header("text/html")},
    put(outh, H),
    accumulate_content(B),
    deliver_accumulated(CliSock, GC, SC),
    done.
    




deliver_403(CliSock, _Req, GC, SC) ->
    B = list_to_binary("<html> <h1> 403 Forbidden</h1> </html>"),
    H = #outh{status = 403,
	      doclose = true,
	      chunked = false,
	      server = yaws:make_server_header(),
	      connection = yaws:make_connection_close_header(true),
	      content_length = yaws:make_content_length_header(size(B)),
	      contlen = size(B),
	      content_type = yaws:make_content_type_header("text/html")},
    put(outh, H),
    accumulate_content(B),
    deliver_accumulated(CliSock, GC, SC),
    done.
    



do_yaws(CliSock, GC, SC, Req, ARG, UT, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    FileAtom = list_to_atom(UT#urltype.fullpath),
    Mtime = mtime(UT#urltype.finfo),
    case ets:lookup(SC#sconf.ets, FileAtom) of
	[{FileAtom, spec, Mtime1, Spec, Es}] when Mtime1 == Mtime,
						  Es == 0 ->
	    deliver_dyn_file(CliSock, GC, SC, Req, Spec, ARG, UT, N);
	Other  ->
	    del_old_files(Other),
	    {ok, [{errors, Errs}| Spec]} = 
		yaws_compile:compile_file(UT#urltype.fullpath, GC, SC),
	    ?Debug("Spec for file ~s is:~n~p~n",[UT#urltype.fullpath, Spec]),
	    ets:insert(SC#sconf.ets, {FileAtom, spec, Mtime, Spec, Errs}),
	    deliver_dyn_file(CliSock, GC, SC, Req, Spec, ARG, UT, N)
    end.


del_old_files([]) ->
    ok;
del_old_files([{_FileAtom, spec, _Mtime1, Spec, _}]) ->
    lists:foreach(
      fun({mod, _, _, _,  Mod, _Func}) ->
	      F="/tmp/yaws/" ++ yaws:to_list(Mod) ++ ".erl",
	      code:purge(Mod),
	      code:purge(Mod),
	      file:delete(F);
	 (_) ->
	      ok
      end, Spec).
		     

get_client_data(CliSock, all, GC, SSlBool) ->
    get_client_data(CliSock, all, cli_recv(CliSock, 4000, GC, SSlBool), 
		    GC, SSlBool);

get_client_data(CliSock, Len, GC, SSlBool) ->
    case cli_recv(CliSock, Len, GC, SSlBool) of
	{ok, B} when size(B) == Len ->
	    B;
	_Other ->
	    ?Debug("get_client_data: ~p~n", [_Other]),
	    exit(normal)
    end.

get_client_data(CliSock, all, {ok, B}, GC, SSlBool) ->
    B2 = get_client_data(CliSock, all, 
			 cli_recv(CliSock, 4000, GC, SSlBool), SSlBool),
    <<B/binary, B2/binary>>;
get_client_data(_CliSock, all, eof, _GC, _) ->
    <<>>.



do_appmod(Mod, FunName, CliSock, GC, SC, Req, ARG, UT, N) ->
    case yaws_call(0, "appmod", Mod, FunName, ARG, GC,SC, N) of
	{streamcontent, MimeType, FirstChunk} ->
	    yaws:outh_set_content_type(MimeType),
	    accumulate_chunk(FirstChunk),
	    deliver_accumulated(CliSock, GC, SC),
	    stream_loop(CliSock, GC, SC),
	    case yaws:outh_get_chunked() of
		true ->
		    gen_tcp_send(CliSock, [crnl(), "0", crnl2()], SC, GC),
		    continue;
		false ->
		    done
	    end;

	{streamcontent_with_size, Sz, MimeType, FirstChunk} ->
	    yaws:outh_set_content_type(MimeType),
	    yaws:outh_set_transfer_encoding_off(),
	    yaws:outh_set_content_length(Sz),
	    accumulate_chunk(FirstChunk),
	    deliver_accumulated(CliSock, GC, SC),
	    stream_loop(CliSock, GC, SC),
	    done;


	_ ->
	    %% finish up
	    deliver_dyn_file(CliSock, GC,SC,Req, UT, [], [], [], ARG,N)
    end.




%% do the header and continue
deliver_dyn_file(CliSock, GC, SC, Req, Specs, ARG, UT, N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    Fd = ut_open(UT),
    Bin = ut_read(Fd),
    close_if_HEAD(Req, fun() ->
			       deliver_accumulated(CliSock,GC,SC),
			       do_tcp_close(CliSock, SC), 
			       throw({ok, 1}) 
		       end),
    deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin, Fd, Specs, ARG, N).



deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin, Fd, [H|T],ARG,N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}, {record, UT,urltype}]),
    ?Debug("deliver_dyn_file: ~p~n", [H]),
    case H of
	{mod, LineNo, YawsFile, NumChars, Mod, out} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    case yaws_call(LineNo, YawsFile, Mod, out, ARG, GC,SC, N) of
		break ->
		    deliver_dyn_file(CliSock, GC, SC, Req,  
				     UT, Bin, Fd, [],ARG,N) ;
		{page, Page} ->
		    {page, Page};
		{streamcontent, MimeType, FirstChunk} ->
		    yaws:outh_set_content_type(MimeType),
		    accumulate_chunk(FirstChunk),
		    deliver_accumulated(CliSock, GC, SC),
		    stream_loop(CliSock, GC, SC),
		    case yaws:outh_get_chunked() of
			true ->
			    gen_tcp_send(CliSock, [crnl(), "0", 
						   crnl2()], SC, GC),
			    continue;
			false ->
			    done
		    end;
		{streamcontent_with_size, Sz, MimeType, FirstChunk} ->
		    yaws:outh_set_content_type(MimeType),
		    yaws:outh_set_transfer_encoding_off(),
		    yaws:outh_set_content_length(Sz),
		    accumulate_chunk(FirstChunk),
		    deliver_accumulated(CliSock, GC, SC),
		    stream_loop(CliSock, GC, SC),
		    done;
		_ ->
		    deliver_dyn_file(CliSock, GC, SC, Req, 
				     UT, Bin2,Fd,T,ARG,0)
	    end;
	{data, 0} ->
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin, Fd,T,ARG,N);
	{data, NumChars} ->
	    {Send, Bin2} = skip_data(Bin, Fd, NumChars),
	    accumulate_chunk(Send),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin2, Fd, T,ARG,N);
	{skip, 0} ->
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin, Fd,T,ARG,N);
	{skip, NumChars} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT,Bin2, Fd, T,ARG,N);
	{error, NumChars, Str} ->
	    {_, Bin2} = skip_data(Bin, Fd, NumChars),
	    accumulate_chunk(Str),
	    deliver_dyn_file(CliSock, GC, SC, Req, UT, Bin2, Fd, T,ARG,N)
    end;

deliver_dyn_file(CliSock, GC, SC, _Req, _UT, _Bin, _Fd, [], _ARG,_N) ->
    ?TC([{record, GC, gconf}, {record, SC, sconf}]),
    ?Debug("deliver_dyn: done~n", []),
    Ret = case yaws:outh_get_chunked() of
	      true ->
		  accumulate_content([crnl(), "0", crnl2()]),
		  continue;
	      false ->
		  done
	  end,
    deliver_accumulated(CliSock, GC, SC),
    Ret.


stream_loop(CliSock, GC, SC) ->
    receive
	{streamcontent, Cont} ->
	    send_streamcontent_chunk(CliSock, GC, SC, Cont),
	    stream_loop(CliSock, GC, SC) ;
	{streamcontent_with_ack, From, Cont} ->	% acknowledge after send
	    send_streamcontent_chunk(CliSock, GC, SC, Cont),
	    From ! {self(), streamcontent_ack},
	    stream_loop(CliSock, GC, SC) ;
	endofstreamcontent ->
	    ok
    after 30000 ->
	    exit(normal)
    end.





send_streamcontent_chunk(CliSock, GC, SC, Chunk) ->
    accumulate_chunk(Chunk),
    Data = erase(acc_content),
    ?Debug("send ~p bytes to ~p ~n", 
	[size(list_to_binary(Data)), CliSock]),
    gen_tcp_send(CliSock, Data, SC, GC).
    

%% what about trailers ??

skip_data(List, Fd, Sz) when list(List) ->
    skip_data(list_to_binary(List), Fd, Sz);
skip_data(Bin, Fd, Sz) when binary(Bin) ->
    ?Debug("Skip data ~p bytes from", [Sz]),
     case  Bin of
	 <<Head:Sz/binary ,Tail/binary>> ->
	     {Head, Tail};
	 _ ->
	     case (catch file:read(Fd, 4000)) of
		 {ok, Bin2} when binary(Bin2) -> 
		     Bin3 = <<Bin/binary, Bin2/binary>>,
		     skip_data(Bin3, Fd, Sz);
		 _Err ->
		     ?Debug("EXIT in skip_data: ~p  ~p ~p~n", [Bin, Sz, _Err]),
		     exit(normal)
	     end
     end;
skip_data({bin, Bin}, _, Sz) ->
    ?Debug("Skip bin data ~p bytes ", [Sz]),
    <<Head:Sz/binary ,Tail/binary>> = Bin,
    {Head, {bin, Tail}};
skip_data({ok, X}, Fd, Sz) ->
    skip_data(X, Fd, Sz).



to_binary(B) when binary(B) ->
    B;
to_binary(L) when list(L) ->
    list_to_binary(L).



accumulate_header(Data) ->
    case get(acc_headers) of
	undefined ->
	    put(acc_headers, [Data, crnl()]);
	List ->
	    put(acc_headers, [List, Data, crnl()])
    end.

accumulate_content(Data) ->
    case get(acc_content) of
	undefined ->
	    put(acc_content, [Data]);
	List ->
	    put(acc_content, [List, Data])
    end.

accumulate_chunk(Data) ->
    case yaws:outh_get_chunked() of
	false ->
	    accumulate_content(Data);
	true ->
	    B = to_binary(Data),
	    case size(B) of
		0 ->
		    skip;
		Len ->
		    CRNL = crnl(),
		    Data2 = [CRNL, yaws:integer_to_hex(Len) , crnl(), B], 
		    accumulate_content(Data2)
	    end
    end.



handle_out_reply(L, LineNo, YawsFile, SC, A) when list (L) ->
    handle_out_reply_l(L, LineNo, YawsFile, SC, A, undefined);

handle_out_reply({html, Html}, _LineNo, _YawsFile, _SC, _A) ->
    accumulate_chunk(Html);

handle_out_reply({ehtml, E}, _LineNo, _YawsFile, SC, A) ->
    case safe_ehtml_expand(E) of
	{ok, Val} ->
	    accumulate_chunk(Val);
	{error, ErrStr} ->
	    handle_crash(A,ErrStr,SC)
    end;

handle_out_reply({content, MimeType, Cont}, _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    accumulate_chunk(Cont);

handle_out_reply({streamcontent, MimeType, First}, 
		 _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    {streamcontent, MimeType, First};

handle_out_reply({streamcontent_with_size, Sz, MimeType, First}, 
		 _LineNo,_YawsFile, _SC, _A) ->
    yaws:outh_set_content_type(MimeType),
    {streamcontent_with_size, Sz, MimeType, First};

handle_out_reply({header, H},  _LineNo, _YawsFile, _SC, _A) ->
    yaws:accumulate_header(H);

handle_out_reply({allheaders, Hs}, _LineNo, _YawsFile, _SC, _A) ->
    yaws:outh_clear_headers(),
    lists:foreach(fun({header, Head}) -> yaws:accumulate_header(Head) end, Hs);

handle_out_reply({status, Code},_LineNo,_YawsFile,_SC,_A) when integer(Code) ->
    yaws:outh_set_status_code(Code);

handle_out_reply({'EXIT', normal}, _LineNo, _YawsFile, _SC, _A) ->
    exit(normal);

handle_out_reply(break, _LineNo, _YawsFile, _SC, _A) ->
    break;

handle_out_reply({redirect_local, Path0}, _LineNo, _YawsFile, SC, A) ->
    Path = case Path0 of
	       {abs_path, P} ->
		   P;
	       P ->
		   P
	   end,
    Scheme = redirect_scheme(SC),
    Arg = hd(A),
    Headers = Arg#arg.headers,
    HostPort = redirect_host(SC, Headers#headers.host),
    Loc = ["Location: ", Scheme, HostPort, Path, "\r\n"],
    OH = get(outh),
    new_redir_h(OH, Loc),
    ok;



handle_out_reply({redirect, URL}, _LineNo, _YawsFile, _SC, _A) ->
    Loc = ["Location: ", URL, "\r\n"],
    OH = get(outh),
    new_redir_h(OH, Loc),
    ok;

handle_out_reply(ok, _LineNo, _YawsFile, _SC, _A) ->
    ok;

handle_out_reply({'EXIT', Err}, LineNo, YawsFile, SC, ArgL) ->
    A = hd(ArgL),
    L = ?F("~n~nERROR erlang  code  crashed:~n "
	   "File: ~s:~w~n"
	   "Reason: ~p~nReq: ~p~n",
	   [YawsFile, LineNo, Err, A#arg.req]),
    handle_crash(A, L, SC);

handle_out_reply({get_more, Cont, State}, _LineNo, _YawsFile, _SC, _A) ->
    {get_more, Cont, State};

handle_out_reply(Reply, LineNo, YawsFile, SC, ArgL) ->
    A = hd(ArgL),
    L =  ?F("yaws code at ~s:~p crashed or "
	    "ret bad val:~p ~nReq: ~p",
	    [YawsFile, LineNo, Reply, A#arg.req]),
    handle_crash(A, L, SC).

    

handle_out_reply_l([Reply|T], LineNo, YawsFile, SC, A, Res) ->		  
    case handle_out_reply(Reply, LineNo, YawsFile, SC, A) of
	{get_more, Cont, State} ->
	    handle_out_reply_l(T, LineNo, YawsFile, SC, A, {get_more, Cont, State});
	break ->
	    break;
	{page, Page} ->
	    {page, Page};
	RetVal ->
	    handle_out_reply_l(T, LineNo, YawsFile, SC, A, RetVal)
    end;
handle_out_reply_l([], _LineNo, _YawsFile, _SC, _A, Res) ->
    Res.



%% Erlang yaws code crashed, display either the
%% actual crash or a custimized error message 

{'fraction_non_alphanumeric': 0.1939851452318847, 'fraction_numerical': 0.006090012821079623, 'mean_word_length': 2.523247663551402, 'pattern_counts': {'":': 4, '<': 81, '<?xml version=': 0, '>': 951, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182687', 'n_tokens_mistral': 38589, 'n_tokens_neox': 34571, 'n_words': 7443}