use LWP::UserAgent;
use WWW::RobotRules;

my $rules = WWW::RobotRules->new('MOMspider/1.0');

my $ua = LWP::UserAgent->new;
$ua->timeout(10);

my $robots_url = 'http://some.place/robots.txt';
my $response = $ua->get($robots_url);
 
if ($response->is_success) {
  my $robots_txt = $response->decoded_content;
  $rules->parse($robots_url, $robots_txt);

  if( $rules->allowed($url) ) {
    ...
  }
} else {
  die "cant fetch $robots_url: " . $response->status_line;
}