>perl -wMstrict -le
"my $s = 'x(y) (a(b)) ()() q (a(b)c()(d(e(f)g))h) q';
 ;;
 our $rx = qr{ \( ([^()]* | (??{ our $rx }))* \) }xms;
 ;;
 my $ry = qr{ ($rx) [^()]* ($rx) }xms;
 ;;
 $s =~ $ry;
 print qq{1st '$1'   3rd '$3'};
 ;;
 print qq{\nfinal regex:};
 print $ry;
"
1st '(y)'   3rd '(a(b))'

final regex:
(?^msx: ((?^msx: \( ([^()]* | (??{ our $rx }))* \) )) [^()]* ((?^msx: \( ([^()]* | (??{ our $rx }))* \) )) )