One thing to realize is that it makes sense to only apply "floor" after applying "sqrt", and that always applying "floor" after doing "sqrt" doesn't eliminate any solutions. This means there are only two operations: taking the faculty of a number, or to take the square root and flooring the number. This also means all intermediate values are integers.
The following algorithm is a breadth-first search, keeping track of which numbers were already calculated. To keep the intermediate values managable, it won't apply faculty on numbers greater than 200 (this is a tight limit, for at least one of the number 1 .. 10, you need the faculty of 200). After finding solutions for the numbers 1 to 10, it prints them.
#!/usr/bin/perl
use strict;
use warnings;
use Math::BigInt;
use constant FAC_MAX => 200;
sub pretty_print;
my $three = {meth => "init",
value => Math::BigInt->new(3),
old_value => Math::BigInt->new(3)};
my @list = ($three);
my %done = map {$_->{value} => $_} @list;
while (@list) {
my $node = shift @list;
for my $meth (qw /bsqrt bfac/) {
next if $meth eq 'bfac' && $node->{value} > FAC_MAX;
my $new_value = $node->{value}->copy->$meth;
unless ($done{$new_value}) {
my $new = {meth => $meth,
value => $new_value,
old_value => $node->{value}};
push @list, $new;
$done{$new_value} = $new;
}
}
my $c = 0;
$done{$_} && $c++ for 1 .. 10;
if ($c >= 10) {last}
}
for my $n (1 .. 10) {
printf "%2d = ", $n;
pretty_print $done{$n};
print "\n";
}
sub pretty_print {
my $n = shift;
my $m = $n->{meth};
if ($m eq "bsqrt") {
print "|sqrt(";
pretty_print($done{$n->{old_value}});
print ")|";
}
elsif ($m eq "bfac") {
print "(";
pretty_print($done{$n->{old_value}});
print ")!";
}
elsif ($m eq "init") {
print $n->{value}
}
else {die}
}
__END__
1 = |sqrt(3)|
2 = |sqrt((3)!)|
3 = 3
4 = |sqrt(|sqrt(|sqrt(|sqrt(|sqrt(|sqrt((|sqrt(|sqrt(|sqrt(|sqrt(|sqr
+t(|sqrt(|sqrt(((|sqrt(|sqrt(((3)!)!)|)|)!)!)|)|)|)|)|)|)|)!)|)|)|)|)|
+)|
5 = |sqrt(|sqrt(((3)!)!)|)|
6 = (3)!
7 = |sqrt(|sqrt(|sqrt(|sqrt(|sqrt(|sqrt((|sqrt(|sqrt(|sqrt(|sqrt((|sq
+rt(((3)!)!)|)!)|)|)|)|)!)|)|)|)|)|)|
8 = |sqrt(|sqrt((|sqrt(|sqrt(|sqrt(|sqrt(|sqrt(|sqrt((|sqrt(|sqrt(|sq
+rt(|sqrt((|sqrt(((3)!)!)|)!)|)|)|)|)!)|)|)|)|)|)|)!)|)|
9 = |sqrt(|sqrt(|sqrt(|sqrt(|sqrt((|sqrt(|sqrt(|sqrt(|sqrt(|sqrt(|sqr
+t(|sqrt(|sqrt((|sqrt((|sqrt(|sqrt((|sqrt(|sqrt(|sqrt(|sqrt(|sqrt(|sqr
+t((|sqrt(|sqrt(|sqrt(|sqrt((|sqrt(((3)!)!)|)!)|)|)|)|)!)|)|)|)|)|)|)!
+)|)|)!)|)!)|)|)|)|)|)|)|)|)!)|)|)|)|)|
10 = |sqrt((|sqrt(|sqrt(((3)!)!)|)|)!)|