Re: Unpredictable output form Text::ExtractWords

[epoptai]

Lingua::EN::Splitter to the rescue:

#!/usr/bin/perl

use strict;
use warnings;
use Lingua::EN::Splitter;

my @strings = (
  q|one two 'three'|,
  q|yes I am 'me'|,
);

my %config = (
  minlen => 3,
  maxlen => 32,
  locale => "en_US.ISO_8859-1",
);

my $split = Lingua::EN::Splitter->new;
my %count;

for my $str (@strings){

  print "$str\n";

  my $words = $split->words($str);

  $count{$_}++ for @$words;

  for my $word (sort @$words) {
    $_ = length $word;
    next if $_ < $config{minlen} or $_ > $config{maxlen};
    print $word, ' -> ', $count{$word}, "\n"
  }
  print '-' x 10, "\n";
}

__END__

Output:

one two 'three'
one -> 1
three -> 1
two -> 1
----------
yes I am 'me'
yes -> 1