use File::Find;

finddepth (\&remove_dir, "$directory");
rmdir ( "$directory" ) or die ("Could not remove $directory");

sub remove_dir
{
  # for a directory, this will be 0
  if ( ! (stat("$File::Find::name"))[7] ) 
  { rmdir("$File::Find::name"); }
  else
  { unlink("$File::Find::name"); }
}