This does the trick. Usage notes at end.

use 5.010;
use autodie;
use common::sense;
use MooseX::Declare;

BEGIN
{
    $CPAN::Retest::AUTHORITY = 'cpan:TOBYINK';
    $CPAN::Retest::VERSION   = '0.001';
}

class CPAN::Retest
{
    use App::Prove qw//;
    use File::Basename qw/fileparse/;
    use File::pushd qw/pushd/;
    use File::Path qw/make_path/;
    use File::Spec qw//;
    use File::Temp qw//;
    use LWP::Simple qw/get/;
    use Module::Manifest qw//;
    use Object::AUTHORITY qw/AUTHORITY/;
        
    has author    => (is => 'ro', isa => 'Str', required => 1);
    has release   => (is => 'ro', isa => 'Str', required => 1);
    has manifest  => (is => 'rw', isa => 'ArrayRef[Str]', lazy => 1, b
+uilder => '_build_manifest');
    has testdir   => (is => 'ro', isa => 'File::Temp::Dir', lazy => 1,
+ builder => '_build_testdir');
    has verbose   => (is => 'ro');
    
    method url_for ($file)
    {
        sprintf(
            'http://api.metacpan.org/source/%s/%s/%s',
            uc $self->author,
            $self->release,
            $file
            );
    }

    method test_files
    {
        grep { m{^t/} } @{ $self->manifest };
    }
    
    method _build_manifest
    {
        my $fh = File::Temp->new;
        binmode( $fh, ":utf8");
        
        print $fh get($self->url_for('MANIFEST'));
        close $fh;
        
        my $manifest = Module::Manifest->new;
        $manifest->open(manifest => $fh->filename);
        return [ $manifest->files ];
    }
    
    method _build_testdir
    {
        my $testdir = File::Temp->newdir;
        
        foreach my $file ($self->test_files)
        {
            my $dest = File::Spec->catfile($testdir->dirname, $file);
            
            my (undef, $d, undef) = fileparse($dest);
            make_path($d);
            
            open my $fh, '>', $dest;
            print $fh get($self->url_for($file));
            close $fh;
        }
        
        return $testdir;
    }
    
    method run
    {
        my $chdir = pushd($self->testdir->dirname);
        my $app = App::Prove->new;
        $app->process_args('t');
        $app->verbose(1) if $self->verbose;
        $app->run;
        
    }    
}
[download]

Usage:

my $test = CPAN::Retest->new(
  author  => 'TOBYINK',
  release => 'Object-AUTHORITY-0.003',
  verbose => 1,
  );
$test->run;
[download]

Update:

This is now available on CPAN: App-Reprove-0.001.