use File::Spec 'catfile';

opendir my $dir_handle, $dir
  or die "Couldn't open '$dir': $!";

my @file_list =  sort { -M $b <=> -M $a }
                 grep -f, readdir $dir_handle;

unlink map catfile($dir, $_), @f[0,1]
  if @file_list > 20;