gravid, welcome to the monastery.
The following provide 4 demonstrations. MCE modules support Perl not built with threads support. The forth demonstration ensures graceful IO from shared storage or from a network-based file system. In that case, MCE does sequential IO among workers. Parallel IO is possible by adding the MCE option: parallel_io => 1, to have workers read simultaneously (random IO).
threads and threads::shared
use strict;
use warnings;
use threads;
use threads::shared;
my @dirs = qw( a b c d e f );
my %hash = map { $_ => shared_clone({}) } @dirs;
my @thrs;
foreach my $dir (@dirs) {
if (-e "$dir/syn.log") {
push @thrs, threads->create(\&process_1_file, $dir, "syn.log");
}
}
$_->join for @thrs;
print $hash{"a"}{"area"}, "\n";
sub process_1_file {
my ($dir, $file) = @_;
open(my $fh, "$dir/$file") or die "cannot open $dir/$file $!";
while (<$fh>) {
chomp;
if ($_ =~ /^area=(.*)/) {
$hash{"$dir"}{"area"} = $1;
}
}
}
MCE::Hobo and MCE::Shared
use strict;
use warnings;
use MCE::Hobo;
use MCE::Shared;
my @dirs = qw( a b c d e f );
my %hash = map { $_ => MCE::Shared->hash } @dirs;
foreach my $dir (@dirs) {
if (-e "$dir/syn.log") {
MCE::Hobo->create(\&process_1_file, $dir, "syn.log");
}
}
MCE::Hobo->waitall;
print $hash{"a"}{"area"}, "\n";
sub process_1_file {
my ($dir, $file) = @_;
open(my $fh, "$dir/$file") or die "cannot open $dir/$file $!";
while (<$fh>) {
chomp;
if ($_ =~ /^area=(.*)/) {
$hash{"$dir"}{"area"} = $1;
}
}
}
MCE::Loop and MCE::Shared
use strict;
use warnings;
use MCE::Loop;
use MCE::Shared;
my @dirs = qw( a b c d e f );
my %hash = map { $_ => MCE::Shared->hash } @dirs;
MCE::Loop->init(
max_workers => $#dirs,
chunk_size => 1
);
mce_loop {
my $dir = $_;
proccess_1_file($dir, "syn.log") if (-e "$dir/syn.log");
} @dirs;
print $hash{"a"}{"area"}, "\n";
sub proccess_1_file {
my ($dir, $file) = @_;
open(my $fh, "$dir/$file") or die "cannot open $dir/$file $!";
while (<$fh>) {
chomp;
if ($_ =~ /^area=(.*)/) {
$hash{"$dir"}{"area"} = $1;
}
}
}
MCE and MCE::Shared
use strict;
use warnings;
use MCE;
use MCE::Shared;
my @dirs = qw( a b c d e f );
my %hash = map { $_ => MCE::Shared->hash } @dirs;
my $mce = MCE->new(
max_workers => 4,
chunk_size => '12m',
use_slurpio => 1,
user_func => sub {
my ($mce, $slurp_ref, $chunk_id) = @_;
my $dir = $mce->user_args()->[0];
open my $MEM_FH, "<", $slurp_ref;
while (<$MEM_FH>) {
chomp;
if ($_ =~ /^area=(.*)/) {
$hash{"$dir"}{"area"} = $1;
}
}
close $MEM_FH;
}
)->spawn;
foreach my $dir (@dirs) {
if (-e "$dir/syn.log") {
$mce->process({ user_args => [ $dir ] }, "$dir/syn.log");
}
}
$mce->shutdown;
print $hash{"a"}{"area"}, "\n";
Warm regards. Mario