First search http://groups.google.com for "LWP Parallel::ForkManager"
I don't see why LWP::Parallel wasn't showing a speed improvement unless you set the max_req too low.
$ua->max_req(30) defaults to 5.
I don't think threads will help you, except to give realtime progress reports for each download.
Try this: (there are similar examples on groups.google).
#!/usr/bin/perl
use Parallel::ForkManager;
use LWP::Simple;
use LWP::UserAgent ;
use HTTP::Status ;
use HTTP::Request ;
%urls = ('drudge'=> 'http://www.drudgereport.com',
'rush' =>'http://www.rushlimbaugh.com/home/today.guest.html',
'yahoo' => 'http://www.yahoo.com',
'cds' => 'http://www.cdsllc.com/',);
foreach $myURL (sort(values(%urls))){
$count++;
print "Count is $count\n";
$document = DOCUMENT_RETRIEVER($myURL);
}
sub DOCUMENT_RETRIEVER{
$myURL=$_[0];
$mit = $myURL;
print "Commencing DOCUMENT_RETRIEVER number $iteration
for $mit\n";
print "Iteration is $iteration and Count is $count\n";
for ($iteration = $count; $iteration <= $count;$iteration++){
$name = $iteration;
print "NAME $name\n" ;
my $pm=new Parallel::ForkManager(30);
$pm->start and next;
print "Starting Child Process $iteration for $mit\n" ;
$ua = LWP::UserAgent->new;
$ua->agent("$0/0.1 " . $ua->agent);
$req = new HTTP::Request 'GET' => "$mit";
$res = $ua->request($req, "$name.html");
print "Process $iteration Complete\n" ;
$pm->finish;
$pm->wait_all_childs;
print "Waiting on children\n";
}
undef $name;
}