my %seen = ();
while(<FILE>) {
    chomp;
    ($host, $grp, $gid, $userid) = split(/:/, $_);
    $group = "$grp:$gid";
    # check to see if we've seen this group/user already
    next if $seen{$group}{$userid}++;
    push @{$table{$group}}, $userid;
}

Prints:
group1:9001:user1,user2,user3,user4,user5
group1:9006:
group2:9002:user1,user2,user3
group3:9003:user1,user2,user4
group4:9004:user1,user2,user5
group5:9005:user1,user2
group6:9006:
7