Automatically reconnect if a ping isn't seen in 300 seconds

pyircbot/irccore.py

@@ -63,21 +63,38 @@ class IRCCore(asynchat.async_chat):
         self.asynmap = {}
     
     def loop(self):
-        asyncore.loop(map=self.asynmap)
+        while self.alive:
+            try:
+                asyncore.loop(map=self.asynmap, timeout=1)
+            except Exception as e:
+                self.log.error("Loop error: %s" % str(e))
+            # Remove from asynmap
+            for key in list(self.asynmap.keys())[:]:
+                del self.asynmap[key]
+            if self.alive:
+                self._connect()
     
-    def kill(self):
-        """Send quit message and close the socket"""
+    def kill(self, message="Help! Another thread is killing me :(", alive=False):
+        """Send quit message, flush queue, and close the socket
+        
+        :param message: Quit message
+        :type message: str
+        :param alive: True causes a reconnect after disconnecting
+        :type alive: bool
+        """
         # Pauses output queue
-        self.outputQueueRunner.paused = True
+        self.outputQueueRunner.paused = not alive
         # Clear any pending messages
         self.outputQueueRunner.clear()
         # Send quit message and flush queue
-        self.act_QUIT("Help! Another thread is killing me :(")
+        self.act_QUIT(message) # TODO will this hang if the socket is having issues?
         self.outputQueueRunner.flush()
         # Signal disconnection
-        self.alive=False
+        self.alive=alive
         # Close socket
+        self.shutdown(SHUT_RDWR)
         self.close()
+        self.log.info("Kill complete")
     
     " Net related code here on down "
     
@@ -133,10 +150,12 @@ class IRCCore(asynchat.async_chat):
             socket_type = socket.AF_INET6
         socketInfo = socket.getaddrinfo(self.server, self.port, socket_type)
         self.create_socket(socket_type, socket.SOCK_STREAM)
-        self.log.debug("Socket created")
+        self.log.debug("Socket created: %s" % self.socket.fileno())
         self.connect(socketInfo[0][4])
         self.log.debug("Connection established")
+        self._fileno = self.socket.fileno()
         self.asynmap[self._fileno] = self # http://willpython.blogspot.com/2010/08/multiple-event-loops-with-asyncore-and.html
+        self.log.info("_connect: Socket map: %s" % str(self.asynmap))
     
     def handle_connect(self):
         """When asynchat indicates our socket is connected, fire the _CONNECT hook"""

pyircbot/modules/PingResponder.py

@@ -7,14 +7,47 @@
 
 """
 
+from time import time,sleep
+from threading import Thread
 from pyircbot.modulebase import ModuleBase,ModuleHook
 
 class PingResponder(ModuleBase):
     def __init__(self, bot, moduleName):
         ModuleBase.__init__(self, bot, moduleName);
+        self.timer = PingRespondTimer(self)
         self.hooks=[ModuleHook("PING", self.pingrespond)]
     def pingrespond(self, args, prefix, trailing):
         """Respond to the PING command"""
         # got a ping? send it right back
         self.bot.act_PONG(trailing)
         self.log.info("%s Responded to a ping: %s" % (self.bot.get_nick(), trailing))
+        self.timer.reset()
+    def ondisable(self):
+        self.timer.disable()
+
+class PingRespondTimer(Thread):
+    "Tracks last ping from server, and reconnects if over a threshold"
+    def __init__(self, master):
+        Thread.__init__(self)
+        self.daemon = True
+        self.alive = True
+        self.master = master
+        self.reset()
+        self.start()
+    
+    def reset(self):
+        "Reset the internal ping timeout counter"
+        self.lastping = time()
+    
+    def disable(self):
+        "Allow the thread to die"
+        self.alive = False
+    
+    def run(self):
+        while self.alive:
+            sleep(5)
+            if time() - self.lastping > 300: #TODO: configurable timeout
+                self.master.log.info("No pings in %s seconds. Reconnecting" % str(time() - self.lastping))
+                self.master.bot.disconnect("Reconnecting...")
+                self.reset()
+        

pyircbot/pyircbot.py

@@ -68,12 +68,29 @@ class PyIRCBot:
     def loop(self):
         self.irc.loop()
     
-    def kill(self, sys_exit=True):
-        """Shut down the bot violently"""
+    def disconnect(self, message, reconnect=True):
+        """Send quit message and disconnect from IRC.
+        
+        :param message: Quit message
+        :type message: str
+        :param reconnect: True causes a reconnection attempt to be made after the disconnect
+        :type reconnect: bool
+        """
+        self.log.info("disconnect")
+        self.irc.kill(message=message, alive=reconnect)
+    
+    def kill(self, sys_exit=True, message="Help! Another thread is killing me :("):
+        """Shut down the bot violently
+        
+        :param sys_exit: True causes sys.exit(0) to be called
+        :type sys_exit: bool
+        :param message: Quit message
+        :type message: str
+        """
         #Close all modules
         self.closeAllModules()
         
-        self.irc.kill()
+        self.irc.kill(message=message, alive=not sys_exit)
         
         if sys_exit:
             sys.exit(0)

pyircbot/rpc.py

@@ -178,7 +178,6 @@ class BotRPC(Thread):
         
         :param message: Quit message
         :type moduleName: str"""
-        self.bot.act_QUIT(message)
-        self.bot.kill()
+        self.bot.kill(message=message)
         return (True, "Shutdown ordered")