#! perl -slw
use strict;
use threads;
use threads::shared;
use Thread::Queue;
use LWP::Simple;


sub worker {
    my( $Q, $curCountRef ) = @_;
    while( my $work = $Q->dequeue ) {
        my( $url, $name ) = split $;, $work;
        my $rc =getstore( $url, $name );
        warn( "Failed to fetch $url: $rc\n" ), next if $rc != RC_OK;
        lock $$curCountRef;
        printf STDERR "%06d:($name) fetched\n", ++$$curCountRef;
    }
}

our $W //= 20;
our $urlFile //= 'urls.txt';

my $Q = new Thread::Queue;
my $curCount :shared = 0;

my @threads = map{
    threads->create( \&worker, $Q, \$curCount );
} 1 .. $W;

open URLS, '<', $urlFile or die "$urlFile : $!";

my $fileNo = 0;
while( my $work = <URLS> ) {
    chomp $work;
    $Q->enqueue( sprintf "%s$;./tmp/saved.%06d", $work, ++$fileNo );
    sleep 1 while $Q->pending > $W;
}
close URLS;

$Q->enqueue( ( undef ) x $W );

$_->join for @threads;