#! /usr/bin/perl -w
use strict;
use warnings;

use LWP;
use threads; 
use Thread::Queue;

my $query ="perl";
my $dataQueue = Thread::Queue->new;
my $threadCount = 0;
my $output_file = "output.txt";
my %html;

my $start = time;
while (<DATA>)
{
    chomp; 
    #s/^\s+//; s/\s+$//;
    #my ($engine, $url) = split /\s+/;
    #next unless $url;
    my $url = $_;
    my $thr = threads->new(\&get_html, $url);
    $thr->detach;
    $threadCount ++;
}

while ($threadCount)
{
    my $url = $dataQueue->dequeue;
    $html{$url} = $dataQueue->dequeue;
    $threadCount --;
}
print "done in " . scalar ( time - $start) . " seconds.";

#print out the html as a sanity check.
open F, "> output.txt" or die "couldn't open output file";
foreach (keys %html) {
	print F "$_:\n" . $html{$_};
}
close F;

sub get_html
{
    #my $engine = shift;
    my $url = shift;
    
    my $ua = LWP::UserAgent->new;
    $ua->agent('Mozilla/5.0');
    $ua->timeout(10);
    $ua->env_proxy;
    
    my $response = $ua->get($url);
    
    if ($response->is_success) {
        $dataQueue->enqueue($url, $response->content);
    }
    else {
        $dataQueue->enqueue($url, $response->message);
    }
}

__DATA__
http://www.google.com 
http://www.yahoo.com 
http://www.amazon.com
http://www.ebay.com
http://www.perlmonks.com
http://news.yahoo.com
http://news.google.com
http://www.msn.com
http://www.slashdot.org
http://www.indymedia.org