use WWW::Mechanize::Polite;

my $SITE = 'http://www.unlocalhost.com';
my $mech = WWW::Mechanize::Polite->new;
$mech->parse_robots("$SITE/robots.txt");

for ('allow.txt', 'disallow.txt') {
   my $url = "$SITE/$_";
   $mech->polite_get($url);
   print "$url:\n", $mech->content ? $mech->content : "denied\n";
}