Interesting question. First thought was full slices. It's faster than grep but still nowhere near splice.

use warnings;
use strict;
use Benchmark qw/cmpthese/;

my @numbers = ( 0..20 );
my $index = 3;
my @expect = ( 0..2,4..20 );
use constant TEST => 0;

cmpthese(-2, {
    splice => sub {
        my @output = @numbers;
        splice @output, $index, 1;
        join("\0", @output) eq join("\0", @expect) or die if TEST;
    },
    grep => sub { # https://www.perlmonks.org/?node_id=11123877
        my @output = @numbers[ grep $_ != $index, 0 .. $#numbers ];
        join("\0", @output) eq join("\0", @expect) or die if TEST;
    },
    slice => sub {
        my @output = @numbers[0..$index-1,$index+1..$#numbers];
        join("\0", @output) eq join("\0", @expect) or die if TEST;
    },
});

__END__

           Rate   grep  slice splice
grep   291881/s     --   -38%   -66%
slice  468110/s    60%     --   -46%
splice 859381/s   194%    84%     --
[download]