my $re;
$re = qr/
    \[ # Opening bracket
    ( # Capture the contents
        [^][]+ # Body of a link
        |
        (??{$re}) # Or recurse
    )
    \] # Closing bracket
    /x;

$k = "a[b[c[d]]";
$k =~ s/$re/<$1>/g;
print $k;