Ignoring/Trapping the DIE signal

[chrism01]

Monks,

I have 2 programs that run as daemons ie 24 x 7 and need to keep doing so, even if MySQL goes away.
Conceptually, this didn't seem too hard at first, but I've discovered that I need to trap perl's internal die signal or it'll break anyway.
Unfortunately, I haven't been able to find a way to do that permanently. All the techniques I've tried have died after about the 3rd time an error occurs.
See code and results below.

#!/usr/bin/perl -w

use locale;
use DBI;
use POSIX;

use strict;             # Enforce declarations

# Declare cfg pkg so we can refer to it anywhere
{
    package cfg;

    # Database handle
    $cfg::dbh = '';
    $cfg::db_retry_cnt = 0;
    $cfg::db_cxn_msg_sent = 0;
}


# Connect to Database
db_connect();
if( !$cfg::dbh )
{
    print "startup connect failed\n";
    exit(1);
}


# Install SIG Handlers
$SIG{__DIE__} = \&do_sig_die; # trap & ignore
#local $SIG{__DIE__} = \&do_sig_die; # trap & ignore
#$SIG{__DIE__} = 'IGNORE'; # trap & ignore
#local $SIG{__DIE__} = 'IGNORE'; # trap & ignore

# Call main sub to process msgs
process_msgs();


#*********************************************************************
+*********
sub process_msgs
{
    my (
        $packet_interval,   # interval between keep alive
        $error_msg          # error msg (if any)
        );

    while(1)
    {
        # Check for DB errors as we go
        $error_msg = "";
        ($packet_interval, $error_msg) =
                        db_get_pkt_interval_type("xxx\@x.com.au");
        if( !$error_msg )
        {
            #DEBUG
            print "no error caught!\n";
        }
        sleep(5);
    }
}
#*********************************************************************
+*********
sub db_get_pkt_interval_type
{
    my (
        $sql,               # SQL code
        $sth,               # SQL statement handle
        $error_msg,         # sql error msg
        $userlogin,         # user's login eg [e-mail]
        $username,
        $isp,
        $interval
        );

    # Get input params
    $userlogin = $_[0];

    # Extract ISP name
    ($username, $isp) = split(/@/, $userlogin);

    # Set SQL cmd
    $error_msg = "";
    $sql = "SELECT 1 ".
           "FROM   sometable ".
           "WHERE  somecol = ".$cfg::dbh->quote($isp);
print "here 1\n";
    # Run the select
    $sth = $cfg::dbh->prepare($sql);
print "here 2\n";

    $sth->execute();
print "here 3\n";
    if ( $sth->errstr )
    {
print "here 4\n";

        # SQL failure; set err msg from DB
        $error_msg = "db_get_pkt_interval(): ".localtime()." ".
                                                        $sth->errstr;

        # Check if real SQL error or DB gone away
        if( $sth->errstr =~ /MySQL server has gone away/ )
        {
print "here 5\n";

            # Try to re-connect for later
            # logging done in called sub
            db_connect();
        }
        else
        {
            # Other SQL error, so log
            print "$error_msg\n";
        }
    }
    else
    {
        # Collect values
        ( $interval ) = $sth->fetchrow_array;
    }
    # Finished txn
    $sth->finish;

    # Return user's interval value
    return($interval, $error_msg);
}
#*********************************************************************
+*********
#
# Function      : db_connect
#
# Description   : Connect to the Database, so we can insert the exchan
+ge info
#
#                 Some SQL calls specifically check for
#                 /MySQL server has gone away/ before checking for any
+ other
#                 SQL error. If found, call here, where a count is kep
+t of
#                 the num of times this happens. An email & sms is
#                 only sent if retry limit has been reached and no pre
+v msg
#                 has been sent. If a successful connection to the DB 
+is made,
#                 the err msg flag and retry cnt are reset.
#
# Params        : none
#
# Returns       : none      dbh stored as global
#
#*********************************************************************
+*********
sub db_connect
{
    my (
        $dsn,       # data source name
        $error_msg  # if any
        );

    # Set up connection string...
    $dsn = "DBI:mysql:database=somedb;host=somehost;port=nnnn";

    # ... and connect
    # NB: Set RaiseError => 0 so we can trap errors manually
    # without breaking the program
    $cfg::dbh = DBI->connect( $dsn, "user", "passwd",
                                {RaiseError => 0, AutoCommit => 1});

    if( $DBI::errstr)
    {
        # FAIL
        $error_msg = "db_connect(): $DBI::errstr";
        print "$error_msg\n";
        $cfg::db_retry_cnt++;
        if( !$cfg::db_cxn_msg_sent &&
            $cfg::db_retry_cnt > 3 )
        {
            print "would have sent email/sms\n";
            $cfg::db_cxn_msg_sent = 1;
        }
    }
    else
    {
        # SUCCESS
        $cfg::db_cxn_msg_sent = 0;
        $cfg::db_retry_cnt = 0;
    }
}
#*********************************************************************
+*********
#
# Function      : do_sig_die
#
# Description   : If we get SIGDIE, trap, log, ignore, re-install.
#                 Basically, some internal process keeps breaking this
#                 prog with SIGDIE. We are not using this signal ourse
+lves;
#                 sigh ...
#
# Params        : none
#
# Returns       : none
#
#*********************************************************************
+*********
sub do_sig_die
{
    # Log it for analysis
    print "SIGDIE caught\n";

    # Defensive programming ...
    $SIG{__DIE__} = \&do_sig_die;
}

Results:

here 1
here 2
here 3
no error caught!
here 1
here 2
here 3
DBD::mysql::st fetchrow_array failed: fetch() without execute() at ./p
+d.pl line 119.
no error caught!
here 1
here 2
DBD::mysql::st execute failed: MySQL server has gone away at ./pd.pl l
+ine 91.
here 3
here 4
here 5
DBI connect('database=ispdata;host=localhost;port=3306','radius',...) 
+failed: Can't connect to local MySQL server through socket '/var/lib/
+mysql/mysql.sock' (2) at ./pd.pl line 162
db_connect(): Can't connect to local MySQL server through socket '/var
+/lib/mysql/mysql.sock' (2)
SIGDIE caught
Can't call method "quote" on an undefined value at ./pd.pl line 83.
prompt$>

I'm also curious that the first error msg is 'fetch() without execute()', yet I don't get an error msg about the preceding execute() failing...
I'd prefer a soln that doesn't involve eval'ing everything, as it's not only messy (lots of this type of code to fix), but also these daemons need to be as fast as reasonably possible.

I've researched some options (as you can see), but they all have more or less the same result.
Also, most articles I've read only intercept the die long enough to do eg print a msg, then they die anyway.
My daemons must not die at all.
I saw a short example about redefining perl's internal die mechanism, but couldn't understand it or at least couldn't make it work: http://www.perlmonks.com/?node_id=554552
The eval option there had the weird effect of not going into the eval block again after the first failure, which is not what I want.
Any simple solns greatly appreciated
Chris

Readmore tags added by GrandFather

[ikegami]

I'd prefer a soln that doesn't involve eval'ing everything, as it's not only messy (lots of this type of code to fix), but also these daemons need to be as fast as reasonably possible.

eval BLOCK (as opposed to eval EXPR) does add any slowdown that an alternative wouldn't add. eval BLOCK is like try BLOCK in other languages (whereas eval EXPR does run-time compilation of Perl code).

The only alternative of which I know is $SIG{__DIE__}

[chrism01]

I tried $SIG{__DIE__} variations as you can see in my code above, but it seems to die eventually anyway, as shown.
I've also tried adding an 'eval {};' around the code above ie 'eval {' just after the $SIG{__DIE__} = .. and then end the block ie '};' at the end of the script, but it fails in almost exactly the same way ie prints pretty much the same msgs, prints 'SIGDIE caught', then dies anyway... grrrr

[ioannis]

__DIE_ handlers are different from other handlers. For one, you cannot "catch it", if you try it will still fly away! What is the purpose of a __DIE__ handler then? There are two reasons:
1. to alter the die message,
2. to goto a subroutine (this time with the handler disabled). But your days are numbered: receive another die and you are dead, once the subroutine finishes your life is also finished.

[sgifford]

The most reliable way to do this is to write a small and very simple wrapper program to run your Perl script, and restart it if it fails. That handles nearly all of the failure modes, including Perl itself crashing. Of course, the wrapper program could itself die, so you might want to put it in /etc/inittab, so that init will automatically restart it if it dies. init could still crash, and then your OS will shut down; you should be able to set up your kernel and/or BIOS and/or watchdog timers to automatically reboot in this event. Of course, a meteor could crash into the building where your server is located, but then your Perl script is probably the least of your concerns. :-)

daemontools is a very convenient and reliable wrapper program to use for automatically restarting your script; it automatically installs itself into /etc/inittab, and so solves a great deal of the problem on its own.

[chrism01]

Actually, I've already got that functionality in cron & startup scripts for fatal errors, but it doesn't help if MySQL has gone away. For 1 of the progs, that's fine, but the main one is a radius (Accounting) pkt ctr/sender, with a lot of ancillary info for each rec stored in memory. If it crashes we'd lose that info, which is the main purpose of the machine.
This prog can manage without MYSQL for a while if needed, but I'd like it to try to reconnect regularly until MySQL comes back.
I'd be surprised if there isn't a way to do this...

[sgifford]

Hrm. You could certainly store the data in stable storage, like with DB_File, though that would have a performance cost. On the other hand, I've never seen a die that eval didn't deal with properly...

[McDarren]

Perhaps you might find this useful? (I did)

Cheers,
Darren :)

[chrism01]

Looks interesting, I'll give it a try...

[chrism01]

Maybe you could clarify that technique a bit. I tried:

use Tie::Constrained;

# Other code as before, with and without SIG{__DIE__}

sub db_connect
{
    my (
        $dsn,       # data source name
        $error_msg  # if any
        );

    # Set up connection string...
    $dsn = "DBI:mysql:database=xxx;host=somehost;port=nn";

    tie $cfg::dbh,
        'Tie::Constrained' =>
                    {
                    test => sub {
                                    $_[0]->isa('DBI::db')
                                    and $_[0]->ping;
                                },
                    value => DBI->connect($dsn,
                                          "user", "passwd",
#'DBI:mysql:','','',
                             {RaiseError => 0, AutoCommit => 1}),
                    fail => sub {
                                    $_[1] = DBI->connect($dsn,
                                                     "user", "passwd",
                                 {RaiseError => 0, AutoCommit => 1});
                            if( $DBI::errstr)
                            {
                                # FAIL
                                $error_msg = "db_connect(): $DBI::errs
+tr";
                                print "$error_msg\n";
                                $cfg::db_retry_cnt++;
                                if( !$cfg::db_cxn_msg_sent &&
                                    $cfg::db_retry_cnt > 3 )
                                {
                                    print "would have sent email/sms\n
+";
                                    $cfg::db_cxn_msg_sent = 1;
                                }
                            }
                            else
                            {
                                # SUCCESS
                                $cfg::db_cxn_msg_sent = 0;
                                $cfg::db_retry_cnt = 0;
                            }
                                }
                    };
    $Tie::Constrained::STRICT = 1;
}

Didn't quite understand the DB def for 'value' being empty in example, but tried it anyway, got 'DBD::mysql::st execute failed: No database selected' at each execute, so tried filling in with real values as above, and got same errors as in orig post, ending with 'Can't call method "quote" on an undefined value at ./pd.pl line 83', then perl died.