http://www.perlmonks.org?node_id=718603

alini76 has asked for the wisdom of the Perl Monks concerning the following question:

Hi,

I wrote the following piece of code:

#!/usr/bin/perl -w use lib qw( ..); use LWP::Debug qw(+ -cons); use LWP::UserAgent; use HTML::Form; use HTTP::Request::Common; my $url = "https://www.infoteldistributors.com/secure/b2blogin.asp"; my $login = "anemail\@yahoo.com"; my $passwd = "apassword"; my $ua = LWP::UserAgent->new; my $response = $ua->get($url); #login if (defined $response) { my @forms = HTML::Form->parse($response); my $form = @forms2; $form->value("EM",$login); $form->value("PW",$passwd); my $afterlogin = $ua->request( $form->click ); #infoteldistributors.com has a redirect immediately after login if($afterlogin->content =~ /<meta http-equiv='refresh' content='0;url= +(^'*)'/) { $redirurl = $1; print "redirect: $redirurl\n"; } my $categpages = $ua->get( "https://www.infoteldistributors.com/$redir +url" ); print "before foreach loop\n"; #extract all categories URLs foreach $1 ( $categpages->content =~ /<a title="^"*" href="http:\/\/(^ +"*)"/g ) { print "$1\n"; #my $uri = URI->new( $str => "$1", $scheme => "http" ); #my $req = HTTP::Request->new( GET => "http://$1" ); #$req->header( 'Accept' => 'text/html' ); ########### HERE IT CRASH ############# my $categResp = $ua->get( "https://$1" ); ########### HERE IT CRASH ############# print $categResp->content; #get category name $categResp->content =~ /<span style="text-transform: uppercase;">(^<*) +</; </span> my $categ = $1; #get products foreach $1 ($categResp->content =~ /(^<*) /g){ </span> my $title = $1; #get description $categResp->content =~ /(^<*)</g; </span> my $desc = $1; print "$categ | $title | $desc\n"; } } } else { #If an error occurs then $content will not be defined. print "Cannot get login page.\n"; }
Unfortunately I get the following output around the line marked in the code above:
LWP::UserAgent::request: () LWP::UserAgent::send_request: GET https://www.infoteldistributors.com/ +applications/category/category_tlc.asp?CatId=35&name=Wireless-Network +ing LWP::UserAgent::_need_proxy: Not proxied LWP::UserAgent::request: Simple response: Not Implemented 501 Protocol scheme 'www.infoteldistributors.com/applications/category +/category_tlc.asp?catid=35&name=wireless-networking' is not supported