From 7b2d36646e5f33aee3f73e1c48ecba3add5b6a07 Mon Sep 17 00:00:00 2001
From: Pieter <TUD205521@tudelft.net>
Date: Fri, 29 Jul 2016 15:29:50 +0200
Subject: [PATCH 01/14] heartbeat method

---
 qcodes/process/heartbeat.py | 34 ++++++++++++++++++++++++++++++++++
 qcodes/process/server.py    |  6 ++++++
 2 files changed, 40 insertions(+)
 create mode 100644 qcodes/process/heartbeat.py

diff --git a/qcodes/process/heartbeat.py b/qcodes/process/heartbeat.py
new file mode 100644
index 000000000000..64bd793e003c
--- /dev/null
+++ b/qcodes/process/heartbeat.py
@@ -0,0 +1,34 @@
+import os
+import mmap
+
+bfile=r'p:\tmp\heartbeat.txt'
+
+def initHeartBeat(bfile, reinit=False):
+    ''' Initialize connection to heartbear for writing, use with setHeartBeat '''
+    if (not os.path.exists(bfile) ) or reinit:
+        f = open(bfile, 'wb')
+        f.write(bytes([99,1])) # heartbeat on
+        f.close()
+    
+    f = open(bfile, 'a+b')
+    f.seek(0)
+    m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_WRITE)
+    m.flush()
+    #print(m[0])
+    #print(m[1])
+    #print('size %d' % m.size() )
+
+    return m
+
+def openHeartBeat(bfile):
+    f = open(bfile, 'r')
+    m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+    return m
+
+def setHeartBeat(m, value):
+    ''' Set heartbeat value '''
+    m[1]=value
+
+def readHeartBeat(m):
+    ''' Return heartbeat value '''
+    return m[1]    
\ No newline at end of file
diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index 7acc408bd720..ff07617b61ff 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -205,6 +205,8 @@ def close(self):
             del self.query_lock
 
 
+import qcodes.process.heartbeat
+
 class BaseServer(NestedAttrAccess):
 
     """
@@ -271,6 +273,7 @@ def __init__(self, query_queue, response_queue, shared_attrs=None):
         self._response_queue = response_queue
         self._shared_attrs = shared_attrs
 
+        self.hb = qcodes.process.heartbeat.openHeartBeat(qcodes.process.heartbeat.bfile)
     def run_event_loop(self):
         """
         The default event loop. When this method returns, the server stops.
@@ -285,6 +288,9 @@ def run_event_loop(self):
         while self.running:
             query = self._query_queue.get(timeout=self.timeout)
             self.process_query(query)
+            if not qcodes.process.heartbeat.readHeartBeat(self.hb):
+                logging.info('no heartbeat, stopping process')
+                self.running=False
 
     def process_query(self, query):
         """

From b5cb597c95f7c93b53bc9a0a4bae76362dd09e11 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Tue, 2 Aug 2016 22:51:04 +0200
Subject: [PATCH 02/14] add config module

---
 qcodes/config.py | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 qcodes/config.py

diff --git a/qcodes/config.py b/qcodes/config.py
new file mode 100644
index 000000000000..3ca348b8e101
--- /dev/null
+++ b/qcodes/config.py
@@ -0,0 +1,3 @@
+
+addzmqlogging = 1
+

From c833c17b5e7b24a5f3200a72ca55ebf88ac3cfb1 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Tue, 2 Aug 2016 22:51:29 +0200
Subject: [PATCH 03/14] add logging module

---
 qcodes/utils/loggingGUI.py | 244 +++++++++++++++++++++++++++++++++++++
 1 file changed, 244 insertions(+)
 create mode 100644 qcodes/utils/loggingGUI.py

diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
new file mode 100644
index 000000000000..2abfc4c7801f
--- /dev/null
+++ b/qcodes/utils/loggingGUI.py
@@ -0,0 +1,244 @@
+"""
+A GUI for multi-processing logging using ZMQ
+
+Code is adapted from https://github.com/zeromq/pyzmq/blob/master/examples/logger/zmqlogger.py
+
+Pieter Eendebak <pieter.eendebak@tno.nl>
+
+"""
+
+#%% Import packages
+import logging
+import os
+import random
+import sys
+import time
+import logging
+import argparse
+
+from qtpy import QtCore
+from qtpy import QtGui
+from qtpy import QtWidgets
+
+
+import zmq
+from zmq.log.handlers import PUBHandler
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', default=1, help="verbosity level")
+parser.add_argument('-l', '--level', default=logging.DEBUG, help="logging level")
+parser.add_argument('-p', '--port', type=int, default=5800, help="zmq port")
+parser.add_argument('-g', '--gui', type=int, default=1, help="show gui")
+args = parser.parse_args()
+
+
+#%% Util functions
+
+
+def static_var(varname, value):
+    """ Helper function to create a static variable """
+    def decorate(func):
+        setattr(func, varname, value)
+        return func
+    return decorate
+
+
+@static_var("time", 0)
+def tprint(string, dt=1, output=False):
+    """ Print progress of a loop every dt seconds """
+    if (time.time() - tprint.time) > dt:
+        print(string)
+        tprint.time = time.time()
+        if output:
+            return True
+        else:
+            return
+    else:
+        if output:
+            return False
+        else:
+            return
+
+#%% Functions for installing the logger
+
+
+import zmq.log.handlers
+
+def removeZMQlogger(name=None):
+    logger=logging.getLogger(name)
+    
+    for h in logger.handlers:
+        if isinstance(h, zmq.log.handlers.PUBHandler):
+            print('removing handler %s' % h)
+            logger.removeHandler(h)
+        
+def installZMQlogger(port=5800, name=None, clear=True, level=logging.INFO):
+    if clear:
+        removeZMQlogger(name)
+        
+    ctx = zmq.Context()
+    pub = ctx.socket(zmq.PUB)
+    pub.setsockopt(zmq.RCVHWM, 10)
+    
+    pub.connect('tcp://127.0.0.1:%i' % port)
+
+    if name is None:
+        rootlogger = logging.getLogger()
+    else:
+        rootlogger = logging.getLogger(name)
+    if level is not None:
+        rootlogger.setLevel(level)
+    handler = PUBHandler(pub)
+    pid=os.getpid()
+    pstr='pid %d: ' % pid
+    handler.formatters = {
+        logging.DEBUG: logging.Formatter(pstr +
+        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
+        logging.INFO: logging.Formatter(pstr + "%(message)s\n"),
+        logging.WARN: logging.Formatter(pstr +
+        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
+        logging.ERROR: logging.Formatter(pstr +
+        "%(levelname)s %(filename)s:%(lineno)d - %(message)s - %(exc_info)s\n"),
+        logging.CRITICAL: logging.Formatter(pstr +
+        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n")}
+        
+    rootlogger.addHandler(handler)
+    logging.debug('installZMQlogger: handler installed') # first message always is discarded
+    return rootlogger
+
+#%%
+
+        
+class HorseLoggingGUI(QtWidgets.QDialog):
+    
+    LOG_LEVELS = dict({logging.DEBUG: 'debug', logging.INFO: 'info',
+              logging.WARN: 'warning', logging.ERROR: 'error', logging.CRITICAL: 'critical'})
+    
+    
+    def __init__( self, parent = None ):
+        super(HorseLoggingGUI, self).__init__(parent)
+
+        self.setWindowTitle('HorseAR logger')
+
+        self.imap=dict((v, k) for k, v in self.LOG_LEVELS.items())
+
+        self._console = QtWidgets.QPlainTextEdit(self)
+        self._console.setMaximumBlockCount(2000)
+        
+        self._button  = QtWidgets.QPushButton(self)
+        self._button.setText('Clear')
+        self._levelBox  = QtWidgets.QComboBox(self)
+        for k in sorted(self.LOG_LEVELS.keys()):
+            print('item %s' % k)
+            val=self.LOG_LEVELS[k]            
+            self._levelBox.insertItem(k, val)
+
+        blayout = QtWidgets.QHBoxLayout()
+        blayout.addWidget(self._button)
+        blayout.addWidget(self._levelBox)
+        self._button.clicked.connect(self.clearMessages)
+        self._levelBox.currentIndexChanged.connect(self.setLevel)
+
+        layout = QtWidgets.QVBoxLayout()
+        layout.addWidget(self._console)
+        layout.addLayout(blayout)
+        self.setLayout(layout)
+
+        self.addMessage('logging started...'+'\n')
+        
+        self._levelBox.setCurrentIndex(1)
+        self.loglevel=logging.INFO
+        
+    def setLevel(self, boxidx):
+        name = self._levelBox.itemText(boxidx)
+        lvl=self.imap.get(name, None)
+        print('set level to %s: %d' % (name, lvl) )
+        if lvl is not None:
+            self.loglevel=lvl
+            
+    def addMessage(self, msg, level=None):    
+        print(level)
+        if level is not None:
+            if level<self.loglevel:
+                return
+        self._console.moveCursor (QtGui.QTextCursor.End);
+        self._console.insertPlainText (msg);
+        self._console.moveCursor (QtGui.QTextCursor.End);
+
+    def clearMessages( self ):
+        ''' Clear the messages in the logging window '''
+        dlg._console.clear()
+        self.addMessage('cleared messages...\n')
+
+
+def qt_logger(port, dlg, level=logging.INFO, verbose=1):
+    ctx = zmq.Context()
+    sub = ctx.socket(zmq.SUB)
+    sub.bind('tcp://127.0.0.1:%i' % port)
+    sub.setsockopt(zmq.SUBSCRIBE, b"")
+    sub.setsockopt(zmq.RCVHWM, 10)   
+    
+    logging.basicConfig(level=level)
+    app=QtWidgets.QApplication.instance()
+    app.processEvents()
+
+    print('connected to port %s' % port)
+    while True:
+        tprint('ZMQ logger: logging...', dt=5)
+        try:
+            level, message  = sub.recv_multipart(zmq.NOBLOCK)      
+            #level, message = sub.recv_multipart()
+            message = message.decode('ascii')
+            if message.endswith('\n'):
+                # trim trailing newline, which will get appended again
+                message = message[:-1]
+            level=level.lower().decode('ascii')
+            log = getattr(logging, level)
+            lvlvalue=dlg.imap.get(level, None)
+
+            #print(lvlvalue)
+            log(message)
+            dlg.addMessage(message+'\n', lvlvalue)
+            
+            app.processEvents()
+            
+            if verbose>=2:
+                print('message: %s (level %s)' % (message, level))
+        except:
+            # no messages in system....
+            app.processEvents()
+            time.sleep(.06)
+            message=''
+            level=None
+            pass
+        
+#%%
+if __name__ == '__main__':
+
+    port=args.port
+    verbose=args.verbose
+    
+    app = None
+    if ( not QtWidgets.QApplication.instance() ):
+        app = QtWidgets.QApplication([])
+    dlg = HorseLoggingGUI()
+    dlg.resize( 800, 400)
+    dlg.show()
+
+    # start the log watcher
+    try:
+        #sub_logger(port, level=args.level, verbose=verbose)
+        qt_logger(port, level=args.level, verbose=verbose, dlg=dlg)
+        pass
+    except KeyboardInterrupt:
+        pass
+
+    if ( app ):
+        app.exec_()
+
+#%% Send message to logger
+if 0:
+    port=5800
+    installZMQlogger(port=port, level=None)
+    logging.warning('test')
+    #log_worker(port=5700, interval=1)

From 198f7798dc2ca11e0d5916c773e75cde636809bf Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Tue, 2 Aug 2016 22:51:45 +0200
Subject: [PATCH 04/14] add zmq logging handler

---
 qcodes/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/qcodes/__init__.py b/qcodes/__init__.py
index f7a65d3adeab..3715e077798f 100644
--- a/qcodes/__init__.py
+++ b/qcodes/__init__.py
@@ -9,7 +9,14 @@
 from qcodes.version import __version__
 from qcodes.process.helpers import set_mp_method
 from qcodes.utils.helpers import in_notebook
+from qcodes import config
 
+if config.addzmqlogging:
+    from qcodes.utils.loggingGUI import installZMQlogger
+    qlogger=installZMQlogger()
+
+
+#if qcodes.config
 # code that should only be imported into the main (notebook) thread
 # in particular, importing matplotlib in the side processes takes a long
 # time and spins up other processes in order to try and get a front end

From 13d46bc4ec8a706fb380b7c3808987d95134347d Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Tue, 2 Aug 2016 23:20:29 +0200
Subject: [PATCH 05/14] make heartbeat code work

---
 qcodes/config.py                 |  6 +++++-
 qcodes/process/heartbeat.py      | 18 +++++++++++++++---
 qcodes/process/server.py         | 17 ++++++++++++++---
 qcodes/tests/instrument_mocks.py |  1 +
 qcodes/utils/loggingGUI.py       |  6 +++---
 5 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/qcodes/config.py b/qcodes/config.py
index 3ca348b8e101..b3dc453264cd 100644
--- a/qcodes/config.py
+++ b/qcodes/config.py
@@ -1,3 +1,7 @@
 
-addzmqlogging = 1
+import os
+import tempfile
 
+addzmqlogging = os.environ.get('ZMQLOGGING', 1)
+
+heartbeatfile  = os.path.join(tempfile.gettempdir(), r'qcodes-heartbeat.bin' )
diff --git a/qcodes/process/heartbeat.py b/qcodes/process/heartbeat.py
index 64bd793e003c..792b3e5a90b3 100644
--- a/qcodes/process/heartbeat.py
+++ b/qcodes/process/heartbeat.py
@@ -1,10 +1,17 @@
 import os
 import mmap
 
-bfile=r'p:\tmp\heartbeat.txt'
+from qcodes import config
+
+
+def makeHeartBeatFile(bfile, reinit=False):
+    if (not os.path.exists(bfile) ) or reinit:
+        f = open(bfile, 'wb')
+        f.write(bytes([99,1])) # heartbeat on
+        f.close()
 
 def initHeartBeat(bfile, reinit=False):
-    ''' Initialize connection to heartbear for writing, use with setHeartBeat '''
+    ''' Initialize connection to heartbeat for writing, use with setHeartBeat '''
     if (not os.path.exists(bfile) ) or reinit:
         f = open(bfile, 'wb')
         f.write(bytes([99,1])) # heartbeat on
@@ -31,4 +38,9 @@ def setHeartBeat(m, value):
 
 def readHeartBeat(m):
     ''' Return heartbeat value '''
-    return m[1]    
\ No newline at end of file
+    return m[1]    
+    
+    
+_bfile=config.heartbeatfile 
+makeHeartBeatFile(_bfile)
+    
\ No newline at end of file
diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index ff07617b61ff..e78c54f4029d 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -5,6 +5,7 @@
 from uuid import uuid4
 import builtins
 import logging
+import time
 
 QUERY_WRITE = 'WRITE'
 QUERY_ASK = 'ASK'
@@ -273,7 +274,7 @@ def __init__(self, query_queue, response_queue, shared_attrs=None):
         self._response_queue = response_queue
         self._shared_attrs = shared_attrs
 
-        self.hb = qcodes.process.heartbeat.openHeartBeat(qcodes.process.heartbeat.bfile)
+        self.hb = qcodes.process.heartbeat.openHeartBeat(qcodes.config.heartbeatfile)
     def run_event_loop(self):
         """
         The default event loop. When this method returns, the server stops.
@@ -285,12 +286,22 @@ def run_event_loop(self):
           it's not by setting `self.running = False`)
         """
         self.running = True
+        logging.info('run_event_loop')
+        
+        self.timeout = 5    # temporary to make heartbeat work
         while self.running:
-            query = self._query_queue.get(timeout=self.timeout)
-            self.process_query(query)
+            try:
+                query = self._query_queue.get(timeout=self.timeout)
+            except mp.queues.Empty:
+                query=None
+            if query is not None:
+                self.process_query(query)
             if not qcodes.process.heartbeat.readHeartBeat(self.hb):
                 logging.info('no heartbeat, stopping process')
                 self.running=False
+            else:
+                logging.info('heartbeat alive... %s' % time.ctime())
+
 
     def process_query(self, query):
         """
diff --git a/qcodes/tests/instrument_mocks.py b/qcodes/tests/instrument_mocks.py
index 685b5ed380bc..f4eb33ba1800 100644
--- a/qcodes/tests/instrument_mocks.py
+++ b/qcodes/tests/instrument_mocks.py
@@ -196,6 +196,7 @@ def __init__(self, name='dummy', gates=['dac1', 'dac2', 'dac3'], **kwargs):
         """
         super().__init__(name, **kwargs)
 
+        logging.info('create DummyInstrument')
         # make gates
         for _, g in enumerate(gates):
             self.add_parameter(g,
diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 2abfc4c7801f..640126151251 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -109,16 +109,16 @@ def installZMQlogger(port=5800, name=None, clear=True, level=logging.INFO):
 #%%
 
         
-class HorseLoggingGUI(QtWidgets.QDialog):
+class zmqLoggingGUI(QtWidgets.QDialog):
     
     LOG_LEVELS = dict({logging.DEBUG: 'debug', logging.INFO: 'info',
               logging.WARN: 'warning', logging.ERROR: 'error', logging.CRITICAL: 'critical'})
     
     
     def __init__( self, parent = None ):
-        super(HorseLoggingGUI, self).__init__(parent)
+        super(zmqLoggingGUI, self).__init__(parent)
 
-        self.setWindowTitle('HorseAR logger')
+        self.setWindowTitle('ZMQ logger')
 
         self.imap=dict((v, k) for k, v in self.LOG_LEVELS.items())
 

From 56bdc4401da7acc3063ede3e1affb2a7465ded65 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 09:53:56 +0200
Subject: [PATCH 06/14] autopep8

---
 qcodes/process/heartbeat.py |  31 +++++----
 qcodes/utils/loggingGUI.py  | 132 ++++++++++++++++++------------------
 2 files changed, 83 insertions(+), 80 deletions(-)

diff --git a/qcodes/process/heartbeat.py b/qcodes/process/heartbeat.py
index 792b3e5a90b3..5bd2b668bdfd 100644
--- a/qcodes/process/heartbeat.py
+++ b/qcodes/process/heartbeat.py
@@ -5,42 +5,45 @@
 
 
 def makeHeartBeatFile(bfile, reinit=False):
-    if (not os.path.exists(bfile) ) or reinit:
+    if (not os.path.exists(bfile)) or reinit:
         f = open(bfile, 'wb')
-        f.write(bytes([99,1])) # heartbeat on
+        f.write(bytes([99, 1]))  # heartbeat on
         f.close()
 
+
 def initHeartBeat(bfile, reinit=False):
     ''' Initialize connection to heartbeat for writing, use with setHeartBeat '''
-    if (not os.path.exists(bfile) ) or reinit:
+    if (not os.path.exists(bfile)) or reinit:
         f = open(bfile, 'wb')
-        f.write(bytes([99,1])) # heartbeat on
+        f.write(bytes([99, 1]))  # heartbeat on
         f.close()
-    
+
     f = open(bfile, 'a+b')
     f.seek(0)
     m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_WRITE)
     m.flush()
-    #print(m[0])
-    #print(m[1])
-    #print('size %d' % m.size() )
+    # print(m[0])
+    # print(m[1])
+    # print('size %d' % m.size() )
 
     return m
 
+
 def openHeartBeat(bfile):
     f = open(bfile, 'r')
     m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
     return m
 
+
 def setHeartBeat(m, value):
     ''' Set heartbeat value '''
-    m[1]=value
+    m[1] = value
+
 
 def readHeartBeat(m):
     ''' Return heartbeat value '''
-    return m[1]    
-    
-    
-_bfile=config.heartbeatfile 
+    return m[1]
+
+
+_bfile = config.heartbeatfile
 makeHeartBeatFile(_bfile)
-    
\ No newline at end of file
diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 640126151251..4897fc7bd9b4 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -10,10 +10,7 @@
 #%% Import packages
 import logging
 import os
-import random
-import sys
 import time
-import logging
 import argparse
 
 from qtpy import QtCore
@@ -26,7 +23,8 @@
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-v', '--verbose', default=1, help="verbosity level")
-parser.add_argument('-l', '--level', default=logging.DEBUG, help="logging level")
+parser.add_argument(
+    '-l', '--level', default=logging.DEBUG, help="logging level")
 parser.add_argument('-p', '--port', type=int, default=5800, help="zmq port")
 parser.add_argument('-g', '--gui', type=int, default=1, help="show gui")
 args = parser.parse_args()
@@ -64,22 +62,24 @@ def tprint(string, dt=1, output=False):
 
 import zmq.log.handlers
 
+
 def removeZMQlogger(name=None):
-    logger=logging.getLogger(name)
-    
+    logger = logging.getLogger(name)
+
     for h in logger.handlers:
         if isinstance(h, zmq.log.handlers.PUBHandler):
             print('removing handler %s' % h)
             logger.removeHandler(h)
-        
+
+
 def installZMQlogger(port=5800, name=None, clear=True, level=logging.INFO):
     if clear:
         removeZMQlogger(name)
-        
+
     ctx = zmq.Context()
     pub = ctx.socket(zmq.PUB)
     pub.setsockopt(zmq.RCVHWM, 10)
-    
+
     pub.connect('tcp://127.0.0.1:%i' % port)
 
     if name is None:
@@ -89,48 +89,48 @@ def installZMQlogger(port=5800, name=None, clear=True, level=logging.INFO):
     if level is not None:
         rootlogger.setLevel(level)
     handler = PUBHandler(pub)
-    pid=os.getpid()
-    pstr='pid %d: ' % pid
+    pid = os.getpid()
+    pstr = 'pid %d: ' % pid
     handler.formatters = {
         logging.DEBUG: logging.Formatter(pstr +
-        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
+                                         "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
         logging.INFO: logging.Formatter(pstr + "%(message)s\n"),
         logging.WARN: logging.Formatter(pstr +
-        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
+                                        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n"),
         logging.ERROR: logging.Formatter(pstr +
-        "%(levelname)s %(filename)s:%(lineno)d - %(message)s - %(exc_info)s\n"),
+                                         "%(levelname)s %(filename)s:%(lineno)d - %(message)s - %(exc_info)s\n"),
         logging.CRITICAL: logging.Formatter(pstr +
-        "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n")}
-        
+                                            "%(levelname)s %(filename)s:%(lineno)d - %(message)s\n")}
+
     rootlogger.addHandler(handler)
-    logging.debug('installZMQlogger: handler installed') # first message always is discarded
+    logging.debug('installZMQlogger: handler installed')
+                  # first message always is discarded
     return rootlogger
 
 #%%
 
-        
+
 class zmqLoggingGUI(QtWidgets.QDialog):
-    
+
     LOG_LEVELS = dict({logging.DEBUG: 'debug', logging.INFO: 'info',
-              logging.WARN: 'warning', logging.ERROR: 'error', logging.CRITICAL: 'critical'})
-    
-    
-    def __init__( self, parent = None ):
+                       logging.WARN: 'warning', logging.ERROR: 'error', logging.CRITICAL: 'critical'})
+
+    def __init__(self, parent=None):
         super(zmqLoggingGUI, self).__init__(parent)
 
         self.setWindowTitle('ZMQ logger')
 
-        self.imap=dict((v, k) for k, v in self.LOG_LEVELS.items())
+        self.imap = dict((v, k) for k, v in self.LOG_LEVELS.items())
 
         self._console = QtWidgets.QPlainTextEdit(self)
         self._console.setMaximumBlockCount(2000)
-        
-        self._button  = QtWidgets.QPushButton(self)
+
+        self._button = QtWidgets.QPushButton(self)
         self._button.setText('Clear')
-        self._levelBox  = QtWidgets.QComboBox(self)
+        self._levelBox = QtWidgets.QComboBox(self)
         for k in sorted(self.LOG_LEVELS.keys()):
             print('item %s' % k)
-            val=self.LOG_LEVELS[k]            
+            val = self.LOG_LEVELS[k]
             self._levelBox.insertItem(k, val)
 
         blayout = QtWidgets.QHBoxLayout()
@@ -144,28 +144,28 @@ def __init__( self, parent = None ):
         layout.addLayout(blayout)
         self.setLayout(layout)
 
-        self.addMessage('logging started...'+'\n')
-        
+        self.addMessage('logging started...' + '\n')
+
         self._levelBox.setCurrentIndex(1)
-        self.loglevel=logging.INFO
-        
+        self.loglevel = logging.INFO
+
     def setLevel(self, boxidx):
         name = self._levelBox.itemText(boxidx)
-        lvl=self.imap.get(name, None)
-        print('set level to %s: %d' % (name, lvl) )
+        lvl = self.imap.get(name, None)
+        print('set level to %s: %d' % (name, lvl))
         if lvl is not None:
-            self.loglevel=lvl
-            
-    def addMessage(self, msg, level=None):    
+            self.loglevel = lvl
+
+    def addMessage(self, msg, level=None):
         print(level)
         if level is not None:
-            if level<self.loglevel:
+            if level < self.loglevel:
                 return
-        self._console.moveCursor (QtGui.QTextCursor.End);
-        self._console.insertPlainText (msg);
-        self._console.moveCursor (QtGui.QTextCursor.End);
+        self._console.moveCursor(QtGui.QTextCursor.End)
+        self._console.insertPlainText(msg)
+        self._console.moveCursor(QtGui.QTextCursor.End)
 
-    def clearMessages( self ):
+    def clearMessages(self):
         ''' Clear the messages in the logging window '''
         dlg._console.clear()
         self.addMessage('cleared messages...\n')
@@ -176,69 +176,69 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
     sub = ctx.socket(zmq.SUB)
     sub.bind('tcp://127.0.0.1:%i' % port)
     sub.setsockopt(zmq.SUBSCRIBE, b"")
-    sub.setsockopt(zmq.RCVHWM, 10)   
-    
+    sub.setsockopt(zmq.RCVHWM, 10)
+
     logging.basicConfig(level=level)
-    app=QtWidgets.QApplication.instance()
+    app = QtWidgets.QApplication.instance()
     app.processEvents()
 
     print('connected to port %s' % port)
     while True:
         tprint('ZMQ logger: logging...', dt=5)
         try:
-            level, message  = sub.recv_multipart(zmq.NOBLOCK)      
-            #level, message = sub.recv_multipart()
+            level, message = sub.recv_multipart(zmq.NOBLOCK)
+            # level, message = sub.recv_multipart()
             message = message.decode('ascii')
             if message.endswith('\n'):
                 # trim trailing newline, which will get appended again
                 message = message[:-1]
-            level=level.lower().decode('ascii')
+            level = level.lower().decode('ascii')
             log = getattr(logging, level)
-            lvlvalue=dlg.imap.get(level, None)
+            lvlvalue = dlg.imap.get(level, None)
 
-            #print(lvlvalue)
+            # print(lvlvalue)
             log(message)
-            dlg.addMessage(message+'\n', lvlvalue)
-            
+            dlg.addMessage(message + '\n', lvlvalue)
+
             app.processEvents()
-            
-            if verbose>=2:
+
+            if verbose >= 2:
                 print('message: %s (level %s)' % (message, level))
         except:
             # no messages in system....
             app.processEvents()
             time.sleep(.06)
-            message=''
-            level=None
+            message = ''
+            level = None
             pass
-        
+
 #%%
 if __name__ == '__main__':
 
-    port=args.port
-    verbose=args.verbose
-    
+    port = args.port
+    verbose = args.verbose
+
     app = None
-    if ( not QtWidgets.QApplication.instance() ):
+    if (not QtWidgets.QApplication.instance()):
         app = QtWidgets.QApplication([])
     dlg = HorseLoggingGUI()
-    dlg.resize( 800, 400)
+    dlg.resize(800, 400)
     dlg.show()
 
     # start the log watcher
     try:
-        #sub_logger(port, level=args.level, verbose=verbose)
+        # sub_logger(port, level=args.level, verbose=verbose)
         qt_logger(port, level=args.level, verbose=verbose, dlg=dlg)
         pass
     except KeyboardInterrupt:
         pass
 
-    if ( app ):
+    if (app):
         app.exec_()
 
 #%% Send message to logger
 if 0:
-    port=5800
+    port = 5800
     installZMQlogger(port=port, level=None)
     logging.warning('test')
-    #log_worker(port=5700, interval=1)
+    # log_worker(port=5700, interval=1)

From 527d19076f3e781c0e9fca7bfb09c1819355985a Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 10:03:32 +0200
Subject: [PATCH 07/14] remove bot warnings

---
 qcodes/utils/loggingGUI.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 4897fc7bd9b4..774c3fcdf0e7 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -13,7 +13,6 @@
 import time
 import argparse
 
-from qtpy import QtCore
 from qtpy import QtGui
 from qtpy import QtWidgets
 
@@ -204,7 +203,7 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
 
             if verbose >= 2:
                 print('message: %s (level %s)' % (message, level))
-        except:
+        except Exception:
             # no messages in system....
             app.processEvents()
             time.sleep(.06)

From fda88e037a6b616a23ed737796fba08f31d34602 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 14:02:52 +0200
Subject: [PATCH 08/14] add kill pid button to GUI

---
 qcodes/utils/loggingGUI.py | 38 ++++++++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 774c3fcdf0e7..01038602c150 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -10,8 +10,10 @@
 #%% Import packages
 import logging
 import os
+import signal
 import time
 import argparse
+import re
 
 from qtpy import QtGui
 from qtpy import QtWidgets
@@ -126,6 +128,9 @@ def __init__(self, parent=None):
 
         self._button = QtWidgets.QPushButton(self)
         self._button.setText('Clear')
+        self._killbutton = QtWidgets.QPushButton(self)
+        self._killbutton.setText('Kill heartbeat')
+
         self._levelBox = QtWidgets.QComboBox(self)
         for k in sorted(self.LOG_LEVELS.keys()):
             print('item %s' % k)
@@ -134,8 +139,10 @@ def __init__(self, parent=None):
 
         blayout = QtWidgets.QHBoxLayout()
         blayout.addWidget(self._button)
+        blayout.addWidget(self._killbutton)
         blayout.addWidget(self._levelBox)
         self._button.clicked.connect(self.clearMessages)
+        self._killbutton.clicked.connect(self.killPID)
         self._levelBox.currentIndexChanged.connect(self.setLevel)
 
         layout = QtWidgets.QVBoxLayout()
@@ -147,6 +154,7 @@ def __init__(self, parent=None):
 
         self._levelBox.setCurrentIndex(1)
         self.loglevel = logging.INFO
+        self.nkill = 0
 
     def setLevel(self, boxidx):
         name = self._levelBox.itemText(boxidx)
@@ -156,7 +164,6 @@ def setLevel(self, boxidx):
             self.loglevel = lvl
 
     def addMessage(self, msg, level=None):
-        print(level)
         if level is not None:
             if level < self.loglevel:
                 return
@@ -166,9 +173,13 @@ def addMessage(self, msg, level=None):
 
     def clearMessages(self):
         ''' Clear the messages in the logging window '''
-        dlg._console.clear()
+        self._console.clear()
         self.addMessage('cleared messages...\n')
 
+    def killPID(self):
+        ''' Clear the messages in the logging window '''
+        self.nkill = 10
+
 
 def qt_logger(port, dlg, level=logging.INFO, verbose=1):
     ctx = zmq.Context()
@@ -195,15 +206,26 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
             log = getattr(logging, level)
             lvlvalue = dlg.imap.get(level, None)
 
-            # print(lvlvalue)
-            log(message)
+            if verbose >= 2:
+                log(message)
             dlg.addMessage(message + '\n', lvlvalue)
 
+            if dlg.nkill > 0:
+                print('check pid')
+                m = re.match('pid (\d*): heartbeat', message)
+                dlg.nkill = dlg.nkill - 1
+                if m is not None:
+                    pid = int(m.group(1))
+                    print('killing pid %d' % pid)
+                    cmd = 'kill %d' % pid
+                    os.kill(pid, signal.SIGKILL)  # or signal.SIGKILL
+                    dlg.addMessage(
+                        'send kill signal to pid %d\n' % pid, logging.CRITICAL)
             app.processEvents()
 
             if verbose >= 2:
                 print('message: %s (level %s)' % (message, level))
-        except Exception:
+        except zmq.error.Again as ex:
             # no messages in system....
             app.processEvents()
             time.sleep(.06)
@@ -220,7 +242,7 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
     app = None
     if (not QtWidgets.QApplication.instance()):
         app = QtWidgets.QApplication([])
-    dlg = HorseLoggingGUI()
+    dlg = zmqLoggingGUI()
     dlg.resize(800, 400)
     dlg.show()
 
@@ -232,8 +254,8 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
     except KeyboardInterrupt:
         pass
 
-    if (app):
-        app.exec_()
+    # if (app):
+    #    app.exec_()
 
 #%% Send message to logger
 if 0:

From 73dcc6b2e52be300f8d54746816e47e4fcbe9d0f Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 14:24:07 +0200
Subject: [PATCH 09/14] make logging work for all servers

---
 qcodes/process/server.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index e78c54f4029d..40bbff2507b9 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -4,6 +4,7 @@
 from traceback import format_exc
 from uuid import uuid4
 import builtins
+import os
 import logging
 import time
 
@@ -12,6 +13,8 @@
 RESPONSE_OK = 'OK'
 RESPONSE_ERROR = 'ERROR'
 
+from qcodes import config
+from qcodes.utils import loggingGUI
 from qcodes.utils.nested_attrs import NestedAttrAccess
 from .qcodes_process import QcodesProcess
 from .helpers import kill_queue
@@ -208,6 +211,7 @@ def close(self):
 
 import qcodes.process.heartbeat
 
+
 class BaseServer(NestedAttrAccess):
 
     """
@@ -274,7 +278,9 @@ def __init__(self, query_queue, response_queue, shared_attrs=None):
         self._response_queue = response_queue
         self._shared_attrs = shared_attrs
 
-        self.hb = qcodes.process.heartbeat.openHeartBeat(qcodes.config.heartbeatfile)
+        self.hb = qcodes.process.heartbeat.openHeartBeat(
+            qcodes.config.heartbeatfile)
+
     def run_event_loop(self):
         """
         The default event loop. When this method returns, the server stops.
@@ -286,23 +292,25 @@ def run_event_loop(self):
           it's not by setting `self.running = False`)
         """
         self.running = True
-        logging.info('run_event_loop')
-        
-        self.timeout = 5    # temporary to make heartbeat work
+
+        if qcodes.config.addzmqlogging:
+            self.timeout = 5    # temporary to make heartbeat work
+            _ = loggingGUI.installZMQlogger()
+            logging.info('run_event_loop')
+
         while self.running:
             try:
                 query = self._query_queue.get(timeout=self.timeout)
             except mp.queues.Empty:
-                query=None
+                query = None
             if query is not None:
                 self.process_query(query)
             if not qcodes.process.heartbeat.readHeartBeat(self.hb):
                 logging.info('no heartbeat, stopping process')
-                self.running=False
+                self.running = False
             else:
                 logging.info('heartbeat alive... %s' % time.ctime())
 
-
     def process_query(self, query):
         """
         Act on one query received through the query queue.

From 4913d9b41cb20fca0a0510f24e833186da22c438 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 14:25:26 +0200
Subject: [PATCH 10/14] fix bot warnings

---
 qcodes/utils/loggingGUI.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 01038602c150..4f697e21252a 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -217,7 +217,6 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
                 if m is not None:
                     pid = int(m.group(1))
                     print('killing pid %d' % pid)
-                    cmd = 'kill %d' % pid
                     os.kill(pid, signal.SIGKILL)  # or signal.SIGKILL
                     dlg.addMessage(
                         'send kill signal to pid %d\n' % pid, logging.CRITICAL)
@@ -225,13 +224,12 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
 
             if verbose >= 2:
                 print('message: %s (level %s)' % (message, level))
-        except zmq.error.Again as ex:
+        except zmq.error.Again:
             # no messages in system....
             app.processEvents()
             time.sleep(.06)
             message = ''
             level = None
-            pass
 
 #%%
 if __name__ == '__main__':

From 8265c90c86b6afb4e12b5349ee595db3ebc82bd1 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 16:15:36 +0200
Subject: [PATCH 11/14] clean up code

---
 qcodes/process/server.py   | 1 -
 qcodes/utils/loggingGUI.py | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index 40bbff2507b9..4c6631970e1e 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -4,7 +4,6 @@
 from traceback import format_exc
 from uuid import uuid4
 import builtins
-import os
 import logging
 import time
 
diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 4f697e21252a..49ac51afeb12 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -246,9 +246,7 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
 
     # start the log watcher
     try:
-        # sub_logger(port, level=args.level, verbose=verbose)
         qt_logger(port, level=args.level, verbose=verbose, dlg=dlg)
-        pass
     except KeyboardInterrupt:
         pass
 
@@ -260,4 +258,3 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
     port = 5800
     installZMQlogger(port=port, level=None)
     logging.warning('test')
-    # log_worker(port=5700, interval=1)

From 9430d1ea0d55d7ee600bb6aceba50081497d4b25 Mon Sep 17 00:00:00 2001
From: Pieter <pieter.eendebak@gmail.com>
Date: Wed, 3 Aug 2016 16:39:07 +0200
Subject: [PATCH 12/14] add name of process to heartbeat

---
 qcodes/process/server.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index 4c6631970e1e..3e9f1abd5cf9 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -209,6 +209,8 @@ def close(self):
 
 
 import qcodes.process.heartbeat
+from multiprocessing import current_process
+
 
 
 class BaseServer(NestedAttrAccess):
@@ -308,7 +310,7 @@ def run_event_loop(self):
                 logging.info('no heartbeat, stopping process')
                 self.running = False
             else:
-                logging.info('heartbeat alive... %s' % time.ctime())
+                logging.info('heartbeat of %s: alive... %s' % (current_process().name, time.ctime()) )
 
     def process_query(self, query):
         """

From 410ed0b0b002f1bdc0347c597618798c74d54cb6 Mon Sep 17 00:00:00 2001
From: Pieter <TUD205521@tudelft.net>
Date: Thu, 4 Aug 2016 11:30:53 +0200
Subject: [PATCH 13/14] fix missing signal on windows

---
 qcodes/utils/loggingGUI.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qcodes/utils/loggingGUI.py b/qcodes/utils/loggingGUI.py
index 49ac51afeb12..fba1958b454e 100644
--- a/qcodes/utils/loggingGUI.py
+++ b/qcodes/utils/loggingGUI.py
@@ -217,7 +217,8 @@ def qt_logger(port, dlg, level=logging.INFO, verbose=1):
                 if m is not None:
                     pid = int(m.group(1))
                     print('killing pid %d' % pid)
-                    os.kill(pid, signal.SIGKILL)  # or signal.SIGKILL
+                    mysignal = getattr(signal, 'SIGKILL', signal.SIGTERM)
+                    os.kill(pid, mysignal)  # or signal.SIGKILL
                     dlg.addMessage(
                         'send kill signal to pid %d\n' % pid, logging.CRITICAL)
             app.processEvents()

From a7567e56457a72f5950e218aedb82f864d81daa9 Mon Sep 17 00:00:00 2001
From: Pieter <TUD205521@tudelft.net>
Date: Thu, 1 Sep 2016 19:22:06 +0200
Subject: [PATCH 14/14] limit output rate

---
 qcodes/process/server.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/qcodes/process/server.py b/qcodes/process/server.py
index 3e9f1abd5cf9..991d7bedacf1 100644
--- a/qcodes/process/server.py
+++ b/qcodes/process/server.py
@@ -299,6 +299,7 @@ def run_event_loop(self):
             _ = loggingGUI.installZMQlogger()
             logging.info('run_event_loop')
 
+        ptime=time.time()
         while self.running:
             try:
                 query = self._query_queue.get(timeout=self.timeout)
@@ -310,7 +311,9 @@ def run_event_loop(self):
                 logging.info('no heartbeat, stopping process')
                 self.running = False
             else:
-                logging.info('heartbeat of %s: alive... %s' % (current_process().name, time.ctime()) )
+                if (time.time()-ptime)>.1:
+                    logging.info('heartbeat of %s: alive... %s' % (current_process().name, time.ctime()) )
+                    ptime=time.time()
 
     def process_query(self, query):
         """