require WWW::RobotRules;
my $user_agent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)";
my $robotsrules = new WWW::RobotRules '$user_agent';
.....
$robotsrules->parse($url, $robots_txt);
if($robotsrules->allowed($url)){
}