Beefy Boxes and Bandwidth Generously Provided by pair Networks
The stupid question is the question not asked

Re: Live re(p)laying of HTTP requests

by knoebi (Friar)
on Jun 27, 2004 at 14:49 UTC ( #369963=note: print w/replies, xml ) Need Help??

in reply to Live re(p)laying of HTTP requests

Update: I re-read your question and unfortunaly my answer invokes a (slow) proxy server, which is not a good solution in a live enviroment (it's only a solution for a test enviroment).

Hi Liz,

i was looking for exactly the same thing as you some time ago. Even if you write you don't want to add the extra complexity of a proxy server, i suggest you give it a try because this is an extremly easy way of creating (recording) HTTP requests. for playback, you don't need a proxy server anyway.

My solution is based on the POE-Web Proxyserver from the POE Cookbook. It just writes all the Request with timestamps to a MySQL DB (via Class::DBI).

#!/usr/bin/perl use warnings; use strict; use POE; use POE::Component::Server::TCP; use POE::Component::Client::HTTP; use POE::Filter::HTTPD; use HTTP::Response; use Data::Dumper; use IO::All; use DB::DBI; use Time::HiRes; # use CGI; use CGI::Session; sub DUMP_REQUESTS () { 0 } sub DUMP_RESPONSES () { 0 } sub LISTEN_PORT () { 8088 } ### Spawn a web client to fetch requests through. POE::Component::Client::HTTP->spawn( Alias => 'ua' ); ### Spawn a web server. # The ClientInput function is called to deal with client input. # ClientInput's callback function will receive entire HTTP requests # because this server uses POE::Filter::HTTPD to parse its input. # # InlineStates let us attach our own events and handlers to a TCP # server. Here we attach a handler for the got_response event, which # will be sent to us by Client::HTTP when it has fetched something. POE::Component::Server::TCP->new ( Alias => "web_server", Port => LISTEN_PORT, ClientFilter => 'POE::Filter::HTTPD', ClientInput => \&handle_http_request, InlineStates => { got_response => \&handle_http_response, }, ); ### Run the proxy until it is done, then exit. print "http proxy started on localhost:".&LISTEN_PORT." [OK]\n"; POE::Kernel->run(); exit 0; ### Handle HTTP requests from the client. Pass them to the HTTP ### client component for further processing. Optionally dump the ### request as text to STDOUT. sub handle_http_request { my ( $kernel, $heap, $request ) = @_[ KERNEL, HEAP, ARG0 ]; # print Data::Dumper::Dumper ($request); # If the request is really a HTTP::Response, then it indicates a # problem parsing the client's request. Send the response back so # the client knows what's happened. if ( $request->isa("HTTP::Response") ) { $heap->{client}->put($request); $kernel->yield("shutdown"); return; } my $tmp_str=$request->as_string(); #print $tmp_str; $tmp_str=~/CARMEN_SID=(\w+)/; $request->{_sid}=$1; $request->{_cgi_session} = new CGI::Session(undef, $request->{_sid +}, {Directory=>'/tmp'}); $request->{_sid} = $request->{_cgi_session}->id; $request->{Time_HiRes_gettimeofday} = [Time::HiRes::gettimeofday]; $request->{Time_HiRes_time} = Time::HiRes::time(); # Client::HTTP doesn't support keep-alives yet. $request->header( "Connection", "close" ); $request->header( "Proxy-Connection", "close" ); $request->remove_header("Keep-Alive"); $kernel->post( "ua" => "request", "got_response", $request ); } ### Handle HTTP responses from the POE::Component::Client::HTTP we've ### spawned at the beginning of the program. Send each response back ### to the client that requested it. Optionally display the response ### as text. sub handle_http_response { my ( $kernel, $heap ) = @_[ KERNEL, HEAP ]; my $http_response = $_[ARG1]->[0]; $http_response->push_header( Set_Cookie => 'CARMEN_SID='.$http_res +ponse->{_request}->{_sid} ); my $response_type = $http_response->content_type(); if ( $response_type =~ /^text/i ) { $http_response->as_string() if DUMP_RESPONSES; } else { print "Response wasn't text.\n" if DUMP_RESPONSES; } my $obj_http_response = DB::http_response->create({ http_response => Data::Dumper::Dumper ($http_response) +, time_request => $http_response->{_request}->{Time_HiRe +s_time}, time_http_response => Time::HiRes::time(), time_interval => Time::HiRes::tv_interval ( $http_resp +onse->{_request}->{Time_HiRes_gettimeofday}, [Time::HiRes::gettimeofd +ay]), sid => $http_response->{_request}->{_sid}, uri => $http_response->request->uri }); $obj_http_response->update(); # Avoid sending the response if the client has gone away. $heap->{client}->put($http_response) if defined $heap->{client}; # Shut down the client's connection when the response is sent. $kernel->yield("shutdown"); }
replaying the requests is really easy, this script will do it:
#!/usr/bin/perl use warnings; use strict; use Data::Dumper; use IO::All; use LWP::UserAgent; use URI::http; use DB::DBI; my @objs = DB::http_response->retrieve_all; foreach my $obj (@objs) { my $VAR1; eval ($obj->get('http_response')); my $ua = LWP::UserAgent->new; my $response = $ua->request($VAR1->{_request}); print $VAR1->{_request}->uri."\n"; }
just in case you're interested in DB::DBI (bad name, i know):
#!/usr/bin/perl use warnings; use strict; package DB::DBI; use base 'Class::DBI'; __PACKAGE__->connection('dbi:mysql:carmen', 'root', ''); package DB::http_response; use base 'DB::DBI'; __PACKAGE__->table('http_response'); __PACKAGE__->columns(All =>qw/http_response_id http_response time_request time_http_response time_interval sid uri /); package DB::test_run; use base 'DB::DBI'; __PACKAGE__->table('test_run'); __PACKAGE__->columns(All =>qw/test_run_id name /);
for me this works extremly well. all our testing by hand is recorded since we started testing over this proxy server. and because its all in perl its very easy to change/add all kind of things, like supporting dynamic ID generation, SID, and so on.



Log In?

What's my password?
Create A New User
Node Status?
node history
Node Type: note [id://369963]
and all is quiet...

How do I use this? | Other CB clients
Other Users?
Others contemplating the Monastery: (2)
As of 2018-02-19 22:26 GMT
Find Nodes?
    Voting Booth?
    When it is dark outside I am happiest to see ...

    Results (266 votes). Check out past polls.