Thanks. Using Module::Refresh directly fixed my issue.
$ cat class.pm
package class;
use 5.014;
use strict;
use warnings;
use autodie;
sub new {bless {}, 'class'}
sub talk {print "6111513\n"}
1;
$ cat module_refresh_code.pl
#!/usr/bin/env perl
use strict;
use warnings;
use autodie;
use 5.014;
use Data::Dumper;
use Module::Refresh;
my $refresher = Module::Refresh->new;
do './class.pm';
class->new->talk;
my $dummy = <>; # we change class.pm meanwhile
$refresher->refresh_module('./class.pm');
class->new->talk;
$ ./module_refresh_code.pl
6111513
8111513