use strict; use warnings; use feature qw(say); use MCE::Loop chunk_size => 1, max_workers => 4; my @files = glob '*.gz'; my %result = mce_loop { my ($mce, $chunk_ref, $chunk_id) = @_; ## $file = $_; same thing when chunk_size => 1 my $file = $chunk_ref->[0]; ## http://www.zlib.net/pigz/ ## For pigz, we want -p1 to run on one core only. ## open my $fh, '-|', 'pigz', '-dc', '-p1', $file or do { ... } open my $fh, '-|', 'gzip', '-dc', $file or do { warn "open error ($file): $!\n"; MCE->next(); }; my $count = 0; while ( my $line = <$fh> ) { $count++; # simulate filtering or processing } close $fh; ## Send output to the manager process. ## Ensures workers do not garble STDOUT. MCE->say("$file: $count lines"); ## Gather key-value pair. MCE->gather($file, $count); } @files; ## Workers may persist after running. Request workers to exit. MCE::Loop->finish(); ## Ditto, same output using gathered data. for my $file (@files) { say "$file: ", $result{$file}, " lines"; }