Package Gnumed :: Package pycommon :: Module gmBackendListener
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmBackendListener

  1  __doc__ = """GNUmed database backend listener. 
  2   
  3  This module implements threaded listening for asynchronuous 
  4  notifications from the database backend. 
  5  """ 
  6  #===================================================================== 
  7  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>" 
  8  __license__ = "GPL v2 or later" 
  9   
 10  import sys 
 11  import time 
 12  import threading 
 13  import select 
 14  import logging 
 15   
 16   
 17  if __name__ == '__main__': 
 18          sys.path.insert(0, '../../') 
 19  from Gnumed.pycommon import gmDispatcher 
 20  from Gnumed.pycommon import gmBorg 
 21   
 22   
 23  _log = logging.getLogger('gm.db') 
 24   
 25   
 26  signals2listen4 = [ 
 27          'db_maintenance_warning',               # warns of impending maintenance and asks for disconnect 
 28          'db_maintenance_disconnect',    # announces a forced disconnect and disconnects 
 29          'gm_table_mod'                                  # sent for any (registered) table modification, payload contains details 
 30  ] 
 31   
 32  #===================================================================== 
33 -class gmBackendListener(gmBorg.cBorg):
34
35 - def __init__(self, conn=None, poll_interval=3):
36 37 try: 38 self.already_inited 39 return 40 except AttributeError: 41 pass 42 43 self.debug = False 44 self.__notifications_received = 0 45 self.__messages_sent = 0 46 47 _log.info('starting backend notifications listener thread') 48 49 # the listener thread will regularly try to acquire 50 # this lock, when it succeeds it will quit 51 self._quit_lock = threading.Lock() 52 # take the lock now so it cannot be taken by the worker 53 # thread until it is released in shutdown() 54 if not self._quit_lock.acquire(0): 55 _log.error('cannot acquire thread-quit lock, aborting') 56 raise EnvironmentError("cannot acquire thread-quit lock") 57 58 self._conn = conn 59 self.backend_pid = self._conn.get_backend_pid() 60 _log.debug('notification listener connection has backend PID [%s]', self.backend_pid) 61 self._conn.set_isolation_level(0) # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT 62 self._cursor = self._conn.cursor() 63 try: 64 self._conn_fd = self._conn.fileno() 65 except AttributeError: 66 self._conn_fd = self._cursor.fileno() 67 self._conn_lock = threading.Lock() # lock for access to connection object 68 69 self.__register_interests() 70 71 # check for messages every 'poll_interval' seconds 72 self._poll_interval = poll_interval 73 self._listener_thread = None 74 self.__start_thread() 75 76 self.already_inited = True
77 78 #------------------------------- 79 # public API 80 #-------------------------------
81 - def shutdown(self):
82 _log.debug('received %s notifications', self.__notifications_received) 83 _log.debug('sent %s messages', self.__messages_sent) 84 85 if self._listener_thread is None: 86 self.__shutdown_connection() 87 return 88 89 _log.info('stopping backend notifications listener thread') 90 self._quit_lock.release() 91 try: 92 # give the worker thread time to terminate 93 self._listener_thread.join(self._poll_interval+2.0) 94 try: 95 if self._listener_thread.isAlive(): 96 _log.error('listener thread still alive after join()') 97 _log.debug('active threads: %s' % threading.enumerate()) 98 except: 99 pass 100 except: 101 print(sys.exc_info()) 102 103 self._listener_thread = None 104 105 try: 106 self.__unregister_unspecific_notifications() 107 except: 108 _log.exception('unable to unregister unspecific notifications') 109 110 self.__shutdown_connection() 111 112 return
113 #------------------------------- 114 # event handlers 115 #------------------------------- 116 # internal helpers 117 #-------------------------------
118 - def __register_interests(self):
119 # determine unspecific notifications 120 self.unspecific_notifications = signals2listen4 121 _log.info('configured unspecific notifications:') 122 _log.info('%s' % self.unspecific_notifications) 123 gmDispatcher.known_signals.extend(self.unspecific_notifications) 124 125 # listen to unspecific notifications 126 self.__register_unspecific_notifications()
127 128 #-------------------------------
130 for sig in self.unspecific_notifications: 131 _log.info('starting to listen for [%s]' % sig) 132 cmd = 'LISTEN "%s"' % sig 133 self._conn_lock.acquire(1) 134 try: 135 self._cursor.execute(cmd) 136 finally: 137 self._conn_lock.release()
138 139 #-------------------------------
141 for sig in self.unspecific_notifications: 142 _log.info('stopping to listen for [%s]' % sig) 143 cmd = 'UNLISTEN "%s"' % sig 144 self._conn_lock.acquire(1) 145 try: 146 self._cursor.execute(cmd) 147 finally: 148 self._conn_lock.release()
149 150 #-------------------------------
151 - def __shutdown_connection(self):
152 _log.debug('shutting down connection with backend PID [%s]', self.backend_pid) 153 self._conn_lock.acquire(1) 154 try: 155 self._conn.rollback() 156 self._conn.close() 157 except: 158 pass # connection can already be closed :-( 159 finally: 160 self._conn_lock.release()
161 162 #-------------------------------
163 - def __start_thread(self):
164 if self._conn is None: 165 raise ValueError("no connection to backend available, useless to start thread") 166 167 self._listener_thread = threading.Thread ( 168 target = self._process_notifications, 169 name = self.__class__.__name__ 170 ) 171 self._listener_thread.setDaemon(True) 172 _log.info('starting listener thread') 173 self._listener_thread.start()
174 175 #------------------------------- 176 # the actual thread code 177 #-------------------------------
178 - def _process_notifications(self):
179 180 # loop until quitting 181 _have_quit_lock = None 182 while not _have_quit_lock: 183 184 # quitting ? 185 if self._quit_lock.acquire(0): 186 break 187 188 # wait at most self._poll_interval for new data 189 self._conn_lock.acquire(1) 190 try: 191 ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0] 192 finally: 193 self._conn_lock.release() 194 195 # any input available ? 196 if len(ready_input_sockets) == 0: 197 # no, select.select() timed out 198 # give others a chance to grab the conn lock (eg listen/unlisten) 199 time.sleep(0.3) 200 continue 201 202 # data available, wait for it to fully arrive 203 self._conn_lock.acquire(1) 204 try: 205 self._conn.poll() 206 finally: 207 self._conn_lock.release() 208 209 # any notifications ? 210 while len(self._conn.notifies) > 0: 211 # if self._quit_lock can be acquired we may be in 212 # __del__ in which case gmDispatcher is not 213 # guaranteed to exist anymore 214 if self._quit_lock.acquire(0): 215 _have_quit_lock = 1 216 break 217 218 self._conn_lock.acquire(1) 219 try: 220 notification = self._conn.notifies.pop() 221 finally: 222 self._conn_lock.release() 223 self.__notifications_received += 1 224 if self.debug: 225 print(notification) 226 _log.debug('#%s: %s (first param is PID of sending backend)', self.__notifications_received, notification) 227 # decode payload 228 payload = notification.payload.split('::') 229 operation = None 230 table = None 231 pk_column_name = None 232 pk_of_row = None 233 pk_identity = None 234 for item in payload: 235 if item.startswith('operation='): 236 operation = item.split('=')[1] 237 if item.startswith('table='): 238 table = item.split('=')[1] 239 if item.startswith('PK name='): 240 pk_column_name = item.split('=')[1] 241 if item.startswith('row PK='): 242 pk_of_row = int(item.split('=')[1]) 243 if item.startswith('person PK='): 244 try: 245 pk_identity = int(item.split('=')[1]) 246 except ValueError: 247 _log.exception('error in change notification trigger') 248 pk_identity = -1 249 # try sending intra-client signals: 250 # 1) generic signal 251 self.__messages_sent += 1 252 try: 253 results = gmDispatcher.send ( 254 signal = notification.channel, 255 originated_in_database = True, 256 listener_pid = self.backend_pid, 257 sending_backend_pid = notification.pid, 258 pk_identity = pk_identity, 259 operation = operation, 260 table = table, 261 pk_column_name = pk_column_name, 262 pk_of_row = pk_of_row, 263 message_index = self.__messages_sent, 264 notification_index = self.__notifications_received 265 ) 266 except: 267 print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid)) 268 print(sys.exc_info()) 269 # 2) dynamically emulated old style table specific signals 270 if table is not None: 271 self.__messages_sent += 1 272 signal = '%s_mod_db' % table 273 _log.debug('emulating old-style table specific signal [%s]', signal) 274 try: 275 results = gmDispatcher.send ( 276 signal = signal, 277 originated_in_database = True, 278 listener_pid = self.backend_pid, 279 sending_backend_pid = notification.pid, 280 pk_identity = pk_identity, 281 operation = operation, 282 table = table, 283 pk_column_name = pk_column_name, 284 pk_of_row = pk_of_row, 285 message_index = self.__messages_sent, 286 notification_index = self.__notifications_received 287 ) 288 except: 289 print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid)) 290 print(sys.exc_info()) 291 292 # there *may* be more pending notifications but 293 # we don't care when quitting 294 if self._quit_lock.acquire(0): 295 _have_quit_lock = 1 296 break 297 298 # exit thread activity 299 return
300 #===================================================================== 301 # main 302 #===================================================================== 303 if __name__ == "__main__": 304 305 if len(sys.argv) < 2: 306 sys.exit() 307 308 if sys.argv[1] not in ['test', 'monitor']: 309 sys.exit() 310 311 312 notifies = 0 313 314 from Gnumed.pycommon import gmPG2, gmI18N 315 from Gnumed.business import gmPerson, gmPersonSearch 316 317 gmI18N.activate_locale() 318 gmI18N.install_domain(domain='gnumed') 319 #-------------------------------
320 - def run_test():
321 322 #------------------------------- 323 def dummy(n): 324 return float(n)*n/float(1+n)
325 #------------------------------- 326 def OnPatientModified(): 327 global notifies 328 notifies += 1 329 sys.stdout.flush() 330 print("\nBackend says: patient data has been modified (%s. notification)" % notifies) 331 #------------------------------- 332 try: 333 n = int(sys.argv[2]) 334 except: 335 print("You can set the number of iterations\nwith the second command line argument") 336 n = 100000 337 338 # try loop without backend listener 339 print("Looping", n, "times through dummy function") 340 i = 0 341 t1 = time.time() 342 while i < n: 343 r = dummy(i) 344 i += 1 345 t2 = time.time() 346 t_nothreads = t2-t1 347 print("Without backend thread, it took", t_nothreads, "seconds") 348 349 listener = gmBackendListener(conn = gmPG2.get_raw_connection()) 350 351 # now try with listener to measure impact 352 print("Now in a new shell connect psql to the") 353 print("database <gnumed_v9> on localhost, return") 354 print("here and hit <enter> to continue.") 355 input('hit <enter> when done starting psql') 356 print("You now have about 30 seconds to go") 357 print("to the psql shell and type") 358 print(" notify patient_changed<enter>") 359 print("several times.") 360 print("This should trigger our backend listening callback.") 361 print("You can also try to stop the demo with Ctrl-C !") 362 363 listener.register_callback('patient_changed', OnPatientModified) 364 365 try: 366 counter = 0 367 while counter < 20: 368 counter += 1 369 time.sleep(1) 370 sys.stdout.flush() 371 print('.') 372 print("Looping",n,"times through dummy function") 373 i = 0 374 t1 = time.time() 375 while i < n: 376 r = dummy(i) 377 i += 1 378 t2 = time.time() 379 t_threaded = t2-t1 380 print("With backend thread, it took", t_threaded, "seconds") 381 print("Difference:", t_threaded-t_nothreads) 382 except KeyboardInterrupt: 383 print("cancelled by user") 384 385 listener.shutdown() 386 listener.unregister_callback('patient_changed', OnPatientModified) 387 #-------------------------------
388 - def run_monitor():
389 390 print("starting up backend notifications monitor") 391 392 def monitoring_callback(*args, **kwargs): 393 try: 394 kwargs['originated_in_database'] 395 print('==> got notification from database "%s":' % kwargs['signal']) 396 except KeyError: 397 print('==> received signal from client: "%s"' % kwargs['signal']) 398 del kwargs['signal'] 399 for key in kwargs.keys(): 400 print(' [%s]: %s' % (key, kwargs[key]))
401 402 gmDispatcher.connect(receiver = monitoring_callback) 403 404 listener = gmBackendListener(conn = gmPG2.get_raw_connection()) 405 print("listening for the following notifications:") 406 print("1) unspecific:") 407 for sig in listener.unspecific_notifications: 408 print(' - %s' % sig) 409 410 while True: 411 pat = gmPersonSearch.ask_for_patient() 412 if pat is None: 413 break 414 print("found patient", pat) 415 gmPerson.set_active_patient(patient=pat) 416 print("now waiting for notifications, hit <ENTER> to select another patient") 417 input() 418 419 print("cleanup") 420 listener.shutdown() 421 422 print("shutting down backend notifications monitor") 423 424 #------------------------------- 425 if sys.argv[1] == 'monitor': 426 run_monitor() 427 else: 428 run_test() 429 430 #===================================================================== 431