I tried another variation for PDL.
Update: Reduce RAM usage by processing a sequence of numbers instead (~ 45MB per worker). For predictable output, this requires seeding the generator per each sequence. Previously, the code spawned 100 workers and 1e7 loop iterations (~ 24GB RAM).
use v5.030;
use PDL;
use MCE 1.895;
CORE::srand(42); # This also, for MCE predictable results
# MCE sets internal seed = CORE::random()
PDL::srandom(42); # PDL::srand(N) v2.062 ~ v2.089
MCE->new(
max_workers => MCE::Util::get_ncpu(),
chunk_size => 1,
init_relay => 0,
posix_exit => 1,
sequence => [ 1, 1000 ],
user_func => sub {
# my ($mce, $seq, $chunk_id) = @_;
my $seq = $_;
my $output = "";
# Worker seeds generator using MCE's seed and wid value.
# For sequence of numbers, compute similarly using $seq value.
my $seed = abs(MCE->seed - ($seq * 100000)) % 1073741780;
# PDL::srand($seed); # PDL v1.062 ~ 1.089
PDL::srandom($seed); # PDL v1.089_01
my $pdl = PDL->random(1e6);
foreach (0 .. $pdl->nelem - 1) {
my $r = $pdl->at($_);
$output .= "$r\n";
}
MCE::relay { print $output; };
}
)->run;
Time to run:
$ time perl test_pdl.pl > out # 17GB file size
PDL->random . . . . . . . . . . . 27.211s
$ wc -l out
1000000000