Implement global file/directory change watch service & implement it for the file-browser.

 Includes inline-watch upon filecount/filelist if a watch_id is given.
 The watch handels are identified by window_id, view_id (both set automatically) and watch_id of choice.
 Upon changes a "watch" message will be send via bus containing further details (action, watched path, watch_id).
 Also prevent multiple filelist updates from running simultaneously and improve overall update triggering.

vispa/controller/filesystem.py

@@ -86,12 +86,15 @@ class FSAjaxController(AbstractController):
 
     @cherrypy.expose
     @cherrypy.tools.ajax()
-    def filecount(self, path):
+    def filecount(self, path, watch_id=None):
         self.release_session()
         fs = self.get('fs')
         self.release_database()
 
-        count = fs.get_file_count(path)
+        count = fs.get_file_count(path, 
+                                   window_id=self.get('window_id'),
+                                   view_id=self.get('view_id'),
+                                   watch_id=watch_id)
         if count < 0:
             raise MessageException("%s does not exist" % path)
         else:
@@ -99,7 +102,7 @@ class FSAjaxController(AbstractController):
 
     @cherrypy.expose
     @cherrypy.tools.ajax(encoded=True)
-    def filelist(self, path, deep=False, filefilter=None, reverse=False):
+    def filelist(self, path, deep=False, filefilter=None, reverse=False, watch_id=None):
         self.release_session()
         fs = self.get('fs')
         self.release_database()
@@ -111,7 +114,10 @@ class FSAjaxController(AbstractController):
         # get the files with the filter
         return fs.get_file_list(path, deep=deep,
                                  filter=filefilter,
-                                 reverse=reverse, encode_json=True)
+                                 reverse=reverse, encode_json=True,
+                                 window_id=self.get('window_id'),
+                                 view_id=self.get('view_id'),
+                                 watch_id=watch_id)
 
     @cherrypy.expose
     @cherrypy.tools.ajax()
@@ -240,3 +246,32 @@ class FSAjaxController(AbstractController):
             return self.success(suggestions=suggestions, encode_json=True)
         except Exception, e:
             return self.fail(msg=str(e), encode_json=True)
+
+    @cherrypy.expose
+    @cherrypy.tools.ajax()
+    def watch(self, path, watch_id):
+        self.release_session()
+        fs = self.get('fs')
+        self.release_database()
+
+        err = fs.watch(path, 
+                          window_id=self.get('window_id'),
+                          view_id=self.get('view_id'),
+                          watch_id=watch_id)
+        if err:
+            raise MessageException(err)
+        return {"success": not err}
+
+    @cherrypy.expose
+    @cherrypy.tools.ajax()
+    def unwatch(self, watch_id=None):
+        self.release_session()
+        fs = self.get('fs')
+        self.release_database()
+        
+        err = fs.unwatch(window_id=self.get('window_id'),
+                          view_id=self.get('view_id'),
+                          watch_id=watch_id)
+        if err:
+            raise MessageException(err)
+        return {"success": not err}

vispa/extensions/file/static/js/base/actions.js

@@ -111,7 +111,7 @@ var FileBaseActions = Class.extend({
             "new_name": newName
           });
           dfd.done(function() {
-            self.FileBase.updateView();
+            self.FileBase.updateView(400); // rename triggers 2 events, let's try to wait for both
           });
         }, {
           defaultValue: name
@@ -166,7 +166,7 @@ var FileBaseActions = Class.extend({
           path: "/"
         });
       }
-      self.FileBase.updateView();
+      //self.FileBase.updateView();
       this.entries = {};
     });
   },
@@ -194,7 +194,7 @@ var FileBaseActions = Class.extend({
       "path": JSON.stringify(self.entries)
     });
     dfd.done(function(response) {
-      self.FileBase.updateView();
+      //self.FileBase.updateView();
       this.entries = {};
     });
   },
@@ -233,7 +233,7 @@ var FileBaseActions = Class.extend({
         paths: JSON.stringify(self.entries),
         name: newName
       }).done(function(result) {
-        self.FileBase.updateView();
+        //self.FileBase.updateView();
       });
     });
 
@@ -328,7 +328,7 @@ var FileBaseActions = Class.extend({
         "name": newName,
       });
       dfd.done(function() {
-        self.FileBase.updateView();
+        //self.FileBase.updateView();
       });
     });
   },
@@ -346,7 +346,7 @@ var FileBaseActions = Class.extend({
         "name": newName,
       });
       dfd.done(function() {
-        self.FileBase.updateView();
+        //self.FileBase.updateView();
       });
     });
   },
@@ -366,7 +366,7 @@ var FileBaseActions = Class.extend({
         url: vispa.url.dynamic('ajax/fs/upload?path=' + path +
           '&_workspaceId=' + String(self.FileBase.instance.getWorkspaceId())),
         done: function(e, data) {
-          self.FileBase.updateView();
+          //self.FileBase.updateView();
           vispa.messenger.info('Upload succeeded', 'glyphicon glyphicon-ok-sign');
         },
         fail: function() {

vispa/extensions/file/static/js/base/base.js

 var FileBase = Class.extend({
 
   init: function(instance) {
+    var self = this;
+    
     this.instance = instance;
     // components
     this.view = new FileBaseView(this);
@@ -20,14 +22,28 @@ var FileBase = Class.extend({
       selectmode: false,
       sort: ["name", "type"],
       reverse: [false, false],
-      lastRefresh: null
+      lastRefresh: null,
+      updateState: 0, // 0: idle, >0: last request, -1: running,
+                      // -2: done & requested again, -3: running & requested again
+      lazyUpdate: false,
     };
-
+    
     // Get the default view from the preferences
     viewstring = vispa.device.hasTouch ? "Table" : this.instance.getPreference("View");
     this.workflow.currentView = viewstring == "Symbol" ? Symbolview : Tableview;
+    
+    // buffered refresh events
+    window.setInterval(function(){
+      if (self.workflow.updateState == -2 || (self.workflow.updateState>0 &&
+        ($.now()-self.workflow.updateState) > (self.workflow.lazyUpdate?2000:0))) {
+          // 200ms since "move from/to" upon rename have a 160ms gap (idk why)
+        self.workflow.updateState = -1
+        self._updateView()
+      }
+    }, 50)
+    
   },
-
+  
   setContent: function(node) {
     this.view.setMainContainer(node);
   },
@@ -47,43 +63,56 @@ var FileBase = Class.extend({
     var viewContainer = this.workflow.currentView.render();
   },
 
-  updateView: function() {
+  updateView: function(additionalDelay) {
+    if (this.workflow.updateState >= 0) {
+      if(additionalDelay !== undefined) { // events may want to wait for more events
+        this.workflow.updateState = 
+          Math.max(this.workflow.updateState, $.now() + additionalDelay);
+      } else { // everyone else wants a fast update
+        this.workflow.updateState = 1;
+      }
+    } else if (this.workflow.updateState == -1) {
+      this.workflow.updateState = -3;
+    }
+  },
+  
+  _updateView: function () {
     var self = this;
-
+    
     // remove the links needed for the preview lightbox 
     $("a[data-lightbox=" + this.view.previewBoxID + "]").remove();
-    if (self.workflow.lastRefresh === null || $.now() - self.workflow.lastRefresh > 500) {
-      this.instance.GET("/ajax/fs/filecount", {
-        path: this.workflow.path
-      }).done(function(res) {
+    this.instance.GET("/ajax/fs/filecount", {
+      path: this.workflow.path,
+      watch_id: '0'
+    }).done(function(res) {
 
-        self.instance.setLoading(true, res.count > 50 ? 0 : null);
+      self.instance.setLoading(true, res.count > 50 ? 0 : null);
 
-        // Check if last request is less than one second away
-        var promise = self.instance.GET("/ajax/fs/filelist", {
-          "path": self.workflow.path
-        });
+      // Check if last request is less than one second away
+      var promise = self.instance.GET("/ajax/fs/filelist", {
+        path: self.workflow.path,
+        watch_id: '0'
+      });
 
-        self.workflow.lastRefresh = $.now();
-        if (self.view.fileContentContainer !== null) {
-          self.selections.unselectAll()
-          self.view.fileContentContainer.empty();
-        };
+      if (self.view.fileContentContainer !== null) {
+        self.selections.unselectAll()
+        self.view.fileContentContainer.empty();
+      };
 
-        promise.done(function(content) {
-          self.workflow.lastRefresh = $.now();
-          if (Object.keys(content.filelist).length == 1 && content.filelist[0].warning !==
-            undefined) {
-            self.instance.alert(content.filelist[0].warning);
-            vispa.messenger.error(content.filelist[0].warning);
-            content.filelist = [];
-          }
-          self.refresh(content);
-        }).fail(function(err) {
-          self.instance.setLoading(false);
-        });
+      promise.done(function(content) {
+        if (Object.keys(content.filelist).length == 1 && content.filelist[0].warning !==
+          undefined) {
+          self.instance.alert(content.filelist[0].warning);
+          vispa.messenger.error(content.filelist[0].warning);
+          content.filelist = [];
+        }
+        self.refresh(content);
+        self.workflow.updateState++;
+      }).fail(function(err) {
+        self.instance.setLoading(false);
+        self.workflow.updateState++;
       });
-    }
+    });
   },
 
   refresh: function(content, sort, reverse, filter) {

vispa/extensions/file/static/js/extension.js

@@ -126,6 +126,8 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
   },
 
   setup: function() {
+    var self = this;
+
     this._super();
     this.setLabel(this.fb.workflow.path, true);
     this.setIcon("vispa-icon vispa-icon-folder");
@@ -138,6 +140,12 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
       $(".tableButton", this._nodes.fastMenu).addClass("disabled");
       $(".symbolButton", this._nodes.fastMenu).removeClass("disabled");
     }
+    
+    // register watch event
+    this.onSocket('watch', function(data){
+      self.fb.updateView(100)
+    })
+    
   },
 
   // applyPreferences is called when there was a change to the preferences
@@ -179,19 +187,25 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
   // the following methods are fired before and after certain events
   // return false in the "before" methods to prevent the event
   onBeforeShow: function() {
-    this.fb.updateView();
+    //this.fb.updateView();
+    this.fb.workflow.lazyUpdate = false;
     return this;
   },
   onAfterShow: function() {
     return this;
   },
   onBeforeHide: function() {
+    this.fb.workflow.lazyUpdate = true;
     return this;
   },
   onAfterHide: function() {
     return this;
   },
   onBeforeClose: function() {
+    this.removeSocketListener('watch')
+    this.GET("/ajax/fs/unwatch", {
+      watch_id: '0'
+    })
     return this;
   },
   onAfterClose: function() {
@@ -245,8 +259,16 @@ var FileSelectorView = vispa.ExtensionView.Dialog.extend({
   },
 
   setup: function() {
+    var self = this;
+    
     this._super();
     this.setLabel("id: " + this.getId());
+    
+    // register watch event
+    this.onSocket('watch', function(data){
+      self.fb.updateView()
+    })
+
   },
 
   render: function(content, footer) {
@@ -331,7 +353,14 @@ var FileSelectorView = vispa.ExtensionView.Dialog.extend({
     self.setLoading(false);
   },
   onBeforeShow: function() {
-    this.fb.updateView();
+    //this.fb.updateView();
+    return this;
+  },
+  onBeforeClose: function() {
+    this.removeSocketListener('watch')
+    this.GET("/ajax/fs/unwatch", {
+      watch_id: '0'
+    })
     return this;
   }
 });

vispa/remote/filesystem.py

@@ -5,6 +5,7 @@ from StringIO import StringIO
 from distutils.spawn import find_executable
 from mimetypes import guess_type
 from zipfile import ZipFile
+from threading import Lock
 import json
 import logging
 import os
@@ -12,6 +13,8 @@ import re
 import shutil
 import stat
 import subprocess
+import fsmonitor
+import vispa
 
 try:
     import Image
@@ -43,6 +46,13 @@ class FileSystem(object):
     def __init__(self):
         # allowed extensions
         self.allowed_extensions = FileSystem.FILE_EXTENSIONS
+        self._monitor_thread = fsmonitor.FSMonitorThread(self._monitor_callback)
+        self._monitor_watches = {}
+        self._monitor_listener = {}
+        self._monitor_lock = Lock()
+    
+    def __del__(self):
+        self.close()
 
     def setup(self, basedir=None):
         if basedir is None:
@@ -57,6 +67,11 @@ class FileSystem(object):
             os.makedirs(self.basedir, 0o700)
             return "Basedir now exists"
 
+    def close(self):
+        if self._monitor_thread:
+            self._monitor_thread.remove_all_watches()
+            self._monitor_thread.running = False
+    
     def get_mime_type(self, filepath):
         filepath = self.expand(filepath)
         mime, encoding = guess_type(filepath)
@@ -93,7 +108,13 @@ class FileSystem(object):
             return None
         return target_type if target_type == type else None
 
-    def get_file_count(self, path):
+    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
+        # inline watch
+        if window_id and view_id and watch_id:
+            if self.watch(path, window_id, view_id, watch_id):
+                pass
+                # return -2 # don't fail atm since it would emit the wrong error message
+        # actual function
         path = self.expand(path)
         if os.path.exists(path):
             return len(os.listdir(path))
@@ -102,7 +123,14 @@ class FileSystem(object):
 
     def get_file_list(self, path, deep=False,
                       filter=None, reverse=False,
-                      hide_hidden=True, encode_json=True):
+                      hide_hidden=True, encode_json=True,
+                      window_id=None, view_id=None, watch_id=None):
+        # inline watch
+        if window_id and view_id and watch_id:
+            if self.watch(path, window_id, view_id, watch_id):
+                pass
+                # return "" # don't fail atm since it would not be caught on the client side
+        # actual function
         filter = map(re.compile, filter) if filter else []
         filelist = []
         path_expand = self.expand(path)
@@ -443,6 +471,93 @@ class FileSystem(object):
         else:
             return self.get_file_content(path)
 
+    def _monitor_callback(self, event):
+        if event.action_name == 'access':
+            return
+        
+        with self._monitor_lock:
+            # inform all listener
+            for combined_id in event.watch.listener:
+                vispa.remote.send_topic(
+                    "extension.%s.socket.watch" % combined_id[1],
+                    window_id=combined_id[0],
+                    data={
+                        "path": event.watch.path,
+                        "action_name": event.action_name,
+                        "watch_id": combined_id[2]
+                    })
+            
+            # cleanup yourself
+            if event.action_name == 'delete self':
+                for combined_id in event.watch.listener:
+                    del self._monitor_listener[combined_id]
+                self._check_watch(event.watch)
+    
+    def _check_watch(self, watch):
+        if len(watch.listener) == 0:
+            del self._monitor_watches[watch.path]
+            self._monitor_thread.remove_watch(watch)
+        
+    def watch(self, path, window_id, view_id, watch_id):
+        # fail if there is no such fie
+        path = self.expand(path)
+        if not os.path.exists(path):
+            return "The file does not exist"
+        combined_id = (window_id, view_id, watch_id)
+        
+        with self._monitor_lock:
+            # watch is in use
+            if combined_id in self._monitor_listener:
+                # has path changed?
+                if self._monitor_listener[combined_id].path == path:
+                    return ""
+                else:
+                    self._monitor_listener[combined_id].listener.remove(combined_id)
+                    self._check_watch(self._monitor_listener[combined_id])
+                    del self._monitor_listener[combined_id]
+            
+            # create or get the desired watch
+            if path not in self._monitor_watches:
+                if os.path.isfile(path):
+                    watch = self._monitor_thread.add_file_watch(path)
+                elif os.path.isdir(path):
+                    watch = self._monitor_thread.add_dir_watch(path)
+                else:
+                    return "This kind of file can't be wachted"
+                watch.listener = []
+                self._monitor_watches[path] = watch
+            else:
+                watch = self._monitor_watches[path]
+            
+            if combined_id in watch.listener:
+                return "This file is already watched" # this should never occur
+            
+            watch.listener.append(combined_id)
+            self._monitor_listener[combined_id] = watch
+
+        return ""
+        
+    def unwatch(self, window_id, view_id, watch_id=None):
+        with self._monitor_lock:
+            if watch_id is None:
+                # will perform poor with lots of watches/listener
+                for path, watch in self._monitor_watches.items():
+                    for combined_id in watch.listener:
+                        if window_id==combined_id[0] and view_id==combined_id[1]:
+                            watch.listener.remove(combined_id)
+                            del self._monitor_listener[combined_id]
+                    
+                    self._check_watch(watch)
+            
+            else:
+                combined_id = (window_id, view_id, watch_id)
+                if combined_id in self._monitor_listener:
+                    self._monitor_listener[combined_id].listener.remove(combined_id)
+                    self._check_watch(self._monitor_listener[combined_id])
+                    del self._monitor_listener[combined_id]
+
+        return ""
+
 
 def string_compare(a, b):
     if a == b:

vispa/remote/fsmonitor/__init__.py

+# Copyright (c) 2010, 2012 Luke McCarthy <luke@iogopro.co.uk>
+#
+# This is free software released under the MIT license.
+# See COPYING file for details, or visit:
+# http://www.opensource.org/licenses/mit-license.php
+#
+# The file is part of FSMonitor, a file-system monitoring library.
+# https://github.com/shaurz/fsmonitor
+
+import sys
+import threading
+import traceback
+from .common import FSEvent, FSMonitorError, FSMonitorOSError
+
+# set to None when unloaded
+module_loaded = True
+
+if sys.platform == "linux2":
+    from .linux import FSMonitor
+elif sys.platform == "win32":
+    from .win32 import FSMonitor
+else:
+    from .polling import FSMonitor
+
+class FSMonitorThread(threading.Thread):
+    def __init__(self, callback=None):
+        threading.Thread.__init__(self)
+        self.monitor = FSMonitor()
+        self.callback = callback
+        self._running = True
+        self._events = []
+        self._events_lock = threading.Lock()
+        self.daemon = True
+        self.start()
+
+    def add_dir_watch(self, path, flags=FSEvent.All, user=None):
+        return self.monitor.add_dir_watch(path, flags=flags, user=user)
+
+    def add_file_watch(self, path, flags=FSEvent.All, user=None):
+        return self.monitor.add_file_watch(path, flags=flags, user=user)
+
+    def remove_watch(self, watch):
+        self.monitor.remove_watch(watch)
+
+    def remove_all_watches(self):
+        self.monitor.remove_all_watches()
+        with self._events_lock:
+            self._events = []
+
+    def run(self):
+        while module_loaded and self._running:
+            try:
+                events = self.monitor.read_events()
+                if self.callback:
+                    for event in events:
+                        self.callback(event)
+                else:
+                    with self._events_lock:
+                        self._events.extend(events)
+            except Exception:
+                print "Exception in FSMonitorThread:\n" + traceback.format_exc()
+
+    def stop(self):
+        if self.monitor.watches:
+            self.remove_all_watches()
+            self._running = False
+
+    def read_events(self):
+        with self._events_lock:
+            events = self._events
+            self._events = []
+            return events
+
+__all__ = (
+    "FSMonitor",
+    "FSMonitorThread",
+    "FSMonitorError",
+    "FSMonitorOSError",
+    "FSEvent",
+)

vispa/remote/fsmonitor/common.py

+# Copyright (c) 2010 Luke McCarthy <luke@iogopro.co.uk>
+#
+# This is free software released under the MIT license.
+# See COPYING file for details, or visit:
+# http://www.opensource.org/licenses/mit-license.php
+#
+# The file is part of FSMonitor, a file-system monitoring library.
+# https://github.com/shaurz/fsmonitor