in reply to Re^4: Converting a parallel-serial shell script
in thread Converting a parallel-serial shell script
Here is your solution rewritten to use fork instead of threads:
use strict; use warnings; use Proc::Fork; use IO::Pipe; use IO::Select; my $kids = 4; # How many children we'll create $SIG{CHLD} = 'IGNORE'; my $sel = new IO::Select(); my @payloadlist; for (my $i=0; $i<@ARGV; $i++) { my $idx = $i % $kids; push @{$payloadlist[$idx]}, $ARGV[$i]; } my $start = localtime(); print "Started $start $$\n"; foreach my $payload (@payloadlist) { my $pipe = new IO::Pipe; child { my $p = $pipe->writer; foreach my $filename (@{$payload}) { my $outFile = toTSV($filename); my $now = localtime(); print "$$ $now - Completed $filename\n"; print $p $outFile."\n"; } exit; }; my $r = $pipe->reader; $sel->add($r); } my $workdone = 0 ; while ( $workdone < @payloadlist ) { while(my @ready = $sel->can_read) { foreach my $fh (@ready) { while (<$fh>) { chomp $_; system("echo importing $_\n"); unlink $_; $workdone++; } $sel->remove($fh); $fh->close; } } } my $end = localtime(); print "Completed $end\n"; sub convert{ $_[0]; } sub toTSV { my $filename = shift; my $outFile = $filename . '.tsv'; open my $fhIn, '<', $filename or warn "$filename : $!" and next; open my $fhOut, '>', $outFile or warn "$outFile : $!" and next; while( <$fhIn> ) { my $tsv = convert( $_ ); print $fhOut $tsv; } close $fhOut; close $fhIn; return $outFile; }
Sure, it is more complicated but it has its advantages too. First, nothing is implicitly shared and second it makes you think on what is the best way to divide the workload. The more equally the workload is divided between the nodes the better.
|
---|
Replies are listed 'Best First'. | |
---|---|
Re^6: Converting a parallel-serial shell script
by BrowserUk (Patriarch) on Sep 26, 2008 at 12:07 UTC | |
by kubrat (Scribe) on Sep 29, 2008 at 10:55 UTC |
In Section
Seekers of Perl Wisdom