Commit c7dfb034 authored by Benjamin Fischer's avatar Benjamin Fischer
Browse files

Implement global file/directory change watch service & implement it for the file-browser.

 Includes inline-watch upon filecount/filelist if a watch_id is given.
 The watch handels are identified by window_id, view_id (both set automatically) and watch_id of choice.
 Upon changes a "watch" message will be send via bus containing further details (action, watched path, watch_id).
 Also prevent multiple filelist updates from running simultaneously and improve overall update triggering.
parent fe63341e
......@@ -86,12 +86,15 @@ class FSAjaxController(AbstractController):
@cherrypy.expose
@cherrypy.tools.ajax()
def filecount(self, path):
def filecount(self, path, watch_id=None):
self.release_session()
fs = self.get('fs')
self.release_database()
count = fs.get_file_count(path)
count = fs.get_file_count(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
if count < 0:
raise MessageException("%s does not exist" % path)
else:
......@@ -99,7 +102,7 @@ class FSAjaxController(AbstractController):
@cherrypy.expose
@cherrypy.tools.ajax(encoded=True)
def filelist(self, path, deep=False, filefilter=None, reverse=False):
def filelist(self, path, deep=False, filefilter=None, reverse=False, watch_id=None):
self.release_session()
fs = self.get('fs')
self.release_database()
......@@ -111,7 +114,10 @@ class FSAjaxController(AbstractController):
# get the files with the filter
return fs.get_file_list(path, deep=deep,
filter=filefilter,
reverse=reverse, encode_json=True)
reverse=reverse, encode_json=True,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
@cherrypy.expose
@cherrypy.tools.ajax()
......@@ -240,3 +246,32 @@ class FSAjaxController(AbstractController):
return self.success(suggestions=suggestions, encode_json=True)
except Exception, e:
return self.fail(msg=str(e), encode_json=True)
@cherrypy.expose
@cherrypy.tools.ajax()
def watch(self, path, watch_id):
self.release_session()
fs = self.get('fs')
self.release_database()
err = fs.watch(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
if err:
raise MessageException(err)
return {"success": not err}
@cherrypy.expose
@cherrypy.tools.ajax()
def unwatch(self, watch_id=None):
self.release_session()
fs = self.get('fs')
self.release_database()
err = fs.unwatch(window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
if err:
raise MessageException(err)
return {"success": not err}
......@@ -111,7 +111,7 @@ var FileBaseActions = Class.extend({
"new_name": newName
});
dfd.done(function() {
self.FileBase.updateView();
self.FileBase.updateView(400); // rename triggers 2 events, let's try to wait for both
});
}, {
defaultValue: name
......@@ -166,7 +166,7 @@ var FileBaseActions = Class.extend({
path: "/"
});
}
self.FileBase.updateView();
//self.FileBase.updateView();
this.entries = {};
});
},
......@@ -194,7 +194,7 @@ var FileBaseActions = Class.extend({
"path": JSON.stringify(self.entries)
});
dfd.done(function(response) {
self.FileBase.updateView();
//self.FileBase.updateView();
this.entries = {};
});
},
......@@ -233,7 +233,7 @@ var FileBaseActions = Class.extend({
paths: JSON.stringify(self.entries),
name: newName
}).done(function(result) {
self.FileBase.updateView();
//self.FileBase.updateView();
});
});
......@@ -328,7 +328,7 @@ var FileBaseActions = Class.extend({
"name": newName,
});
dfd.done(function() {
self.FileBase.updateView();
//self.FileBase.updateView();
});
});
},
......@@ -346,7 +346,7 @@ var FileBaseActions = Class.extend({
"name": newName,
});
dfd.done(function() {
self.FileBase.updateView();
//self.FileBase.updateView();
});
});
},
......@@ -366,7 +366,7 @@ var FileBaseActions = Class.extend({
url: vispa.url.dynamic('ajax/fs/upload?path=' + path +
'&_workspaceId=' + String(self.FileBase.instance.getWorkspaceId())),
done: function(e, data) {
self.FileBase.updateView();
//self.FileBase.updateView();
vispa.messenger.info('Upload succeeded', 'glyphicon glyphicon-ok-sign');
},
fail: function() {
......
var FileBase = Class.extend({
init: function(instance) {
var self = this;
this.instance = instance;
// components
this.view = new FileBaseView(this);
......@@ -20,14 +22,28 @@ var FileBase = Class.extend({
selectmode: false,
sort: ["name", "type"],
reverse: [false, false],
lastRefresh: null
lastRefresh: null,
updateState: 0, // 0: idle, >0: last request, -1: running,
// -2: done & requested again, -3: running & requested again
lazyUpdate: false,
};
// Get the default view from the preferences
viewstring = vispa.device.hasTouch ? "Table" : this.instance.getPreference("View");
this.workflow.currentView = viewstring == "Symbol" ? Symbolview : Tableview;
// buffered refresh events
window.setInterval(function(){
if (self.workflow.updateState == -2 || (self.workflow.updateState>0 &&
($.now()-self.workflow.updateState) > (self.workflow.lazyUpdate?2000:0))) {
// 200ms since "move from/to" upon rename have a 160ms gap (idk why)
self.workflow.updateState = -1
self._updateView()
}
}, 50)
},
setContent: function(node) {
this.view.setMainContainer(node);
},
......@@ -47,43 +63,56 @@ var FileBase = Class.extend({
var viewContainer = this.workflow.currentView.render();
},
updateView: function() {
updateView: function(additionalDelay) {
if (this.workflow.updateState >= 0) {
if(additionalDelay !== undefined) { // events may want to wait for more events
this.workflow.updateState =
Math.max(this.workflow.updateState, $.now() + additionalDelay);
} else { // everyone else wants a fast update
this.workflow.updateState = 1;
}
} else if (this.workflow.updateState == -1) {
this.workflow.updateState = -3;
}
},
_updateView: function () {
var self = this;
// remove the links needed for the preview lightbox
$("a[data-lightbox=" + this.view.previewBoxID + "]").remove();
if (self.workflow.lastRefresh === null || $.now() - self.workflow.lastRefresh > 500) {
this.instance.GET("/ajax/fs/filecount", {
path: this.workflow.path
}).done(function(res) {
this.instance.GET("/ajax/fs/filecount", {
path: this.workflow.path,
watch_id: '0'
}).done(function(res) {
self.instance.setLoading(true, res.count > 50 ? 0 : null);
self.instance.setLoading(true, res.count > 50 ? 0 : null);
// Check if last request is less than one second away
var promise = self.instance.GET("/ajax/fs/filelist", {
"path": self.workflow.path
});
// Check if last request is less than one second away
var promise = self.instance.GET("/ajax/fs/filelist", {
path: self.workflow.path,
watch_id: '0'
});
self.workflow.lastRefresh = $.now();
if (self.view.fileContentContainer !== null) {
self.selections.unselectAll()
self.view.fileContentContainer.empty();
};
if (self.view.fileContentContainer !== null) {
self.selections.unselectAll()
self.view.fileContentContainer.empty();
};
promise.done(function(content) {
self.workflow.lastRefresh = $.now();
if (Object.keys(content.filelist).length == 1 && content.filelist[0].warning !==
undefined) {
self.instance.alert(content.filelist[0].warning);
vispa.messenger.error(content.filelist[0].warning);
content.filelist = [];
}
self.refresh(content);
}).fail(function(err) {
self.instance.setLoading(false);
});
promise.done(function(content) {
if (Object.keys(content.filelist).length == 1 && content.filelist[0].warning !==
undefined) {
self.instance.alert(content.filelist[0].warning);
vispa.messenger.error(content.filelist[0].warning);
content.filelist = [];
}
self.refresh(content);
self.workflow.updateState++;
}).fail(function(err) {
self.instance.setLoading(false);
self.workflow.updateState++;
});
}
});
},
refresh: function(content, sort, reverse, filter) {
......
......@@ -126,6 +126,8 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
},
setup: function() {
var self = this;
this._super();
this.setLabel(this.fb.workflow.path, true);
this.setIcon("vispa-icon vispa-icon-folder");
......@@ -138,6 +140,12 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
$(".tableButton", this._nodes.fastMenu).addClass("disabled");
$(".symbolButton", this._nodes.fastMenu).removeClass("disabled");
}
// register watch event
this.onSocket('watch', function(data){
self.fb.updateView(100)
})
},
// applyPreferences is called when there was a change to the preferences
......@@ -179,19 +187,25 @@ var FileBrowserView = vispa.ExtensionView.Center.extend({
// the following methods are fired before and after certain events
// return false in the "before" methods to prevent the event
onBeforeShow: function() {
this.fb.updateView();
//this.fb.updateView();
this.fb.workflow.lazyUpdate = false;
return this;
},
onAfterShow: function() {
return this;
},
onBeforeHide: function() {
this.fb.workflow.lazyUpdate = true;
return this;
},
onAfterHide: function() {
return this;
},
onBeforeClose: function() {
this.removeSocketListener('watch')
this.GET("/ajax/fs/unwatch", {
watch_id: '0'
})
return this;
},
onAfterClose: function() {
......@@ -245,8 +259,16 @@ var FileSelectorView = vispa.ExtensionView.Dialog.extend({
},
setup: function() {
var self = this;
this._super();
this.setLabel("id: " + this.getId());
// register watch event
this.onSocket('watch', function(data){
self.fb.updateView()
})
},
render: function(content, footer) {
......@@ -331,7 +353,14 @@ var FileSelectorView = vispa.ExtensionView.Dialog.extend({
self.setLoading(false);
},
onBeforeShow: function() {
this.fb.updateView();
//this.fb.updateView();
return this;
},
onBeforeClose: function() {
this.removeSocketListener('watch')
this.GET("/ajax/fs/unwatch", {
watch_id: '0'
})
return this;
}
});
......
......@@ -5,6 +5,7 @@ from StringIO import StringIO
from distutils.spawn import find_executable
from mimetypes import guess_type
from zipfile import ZipFile
from threading import Lock
import json
import logging
import os
......@@ -12,6 +13,8 @@ import re
import shutil
import stat
import subprocess
import fsmonitor
import vispa
try:
import Image
......@@ -43,6 +46,13 @@ class FileSystem(object):
def __init__(self):
# allowed extensions
self.allowed_extensions = FileSystem.FILE_EXTENSIONS
self._monitor_thread = fsmonitor.FSMonitorThread(self._monitor_callback)
self._monitor_watches = {}
self._monitor_listener = {}
self._monitor_lock = Lock()
def __del__(self):
self.close()
def setup(self, basedir=None):
if basedir is None:
......@@ -57,6 +67,11 @@ class FileSystem(object):
os.makedirs(self.basedir, 0o700)
return "Basedir now exists"
def close(self):
if self._monitor_thread:
self._monitor_thread.remove_all_watches()
self._monitor_thread.running = False
def get_mime_type(self, filepath):
filepath = self.expand(filepath)
mime, encoding = guess_type(filepath)
......@@ -93,7 +108,13 @@ class FileSystem(object):
return None
return target_type if target_type == type else None
def get_file_count(self, path):
def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
# inline watch
if window_id and view_id and watch_id:
if self.watch(path, window_id, view_id, watch_id):
pass
# return -2 # don't fail atm since it would emit the wrong error message
# actual function
path = self.expand(path)
if os.path.exists(path):
return len(os.listdir(path))
......@@ -102,7 +123,14 @@ class FileSystem(object):
def get_file_list(self, path, deep=False,
filter=None, reverse=False,
hide_hidden=True, encode_json=True):
hide_hidden=True, encode_json=True,
window_id=None, view_id=None, watch_id=None):
# inline watch
if window_id and view_id and watch_id:
if self.watch(path, window_id, view_id, watch_id):
pass
# return "" # don't fail atm since it would not be caught on the client side
# actual function
filter = map(re.compile, filter) if filter else []
filelist = []
path_expand = self.expand(path)
......@@ -443,6 +471,93 @@ class FileSystem(object):
else:
return self.get_file_content(path)
def _monitor_callback(self, event):
if event.action_name == 'access':
return
with self._monitor_lock:
# inform all listener
for combined_id in event.watch.listener:
vispa.remote.send_topic(
"extension.%s.socket.watch" % combined_id[1],
window_id=combined_id[0],
data={
"path": event.watch.path,
"action_name": event.action_name,
"watch_id": combined_id[2]
})
# cleanup yourself
if event.action_name == 'delete self':
for combined_id in event.watch.listener:
del self._monitor_listener[combined_id]
self._check_watch(event.watch)
def _check_watch(self, watch):
if len(watch.listener) == 0:
del self._monitor_watches[watch.path]
self._monitor_thread.remove_watch(watch)
def watch(self, path, window_id, view_id, watch_id):
# fail if there is no such fie
path = self.expand(path)
if not os.path.exists(path):
return "The file does not exist"
combined_id = (window_id, view_id, watch_id)
with self._monitor_lock:
# watch is in use
if combined_id in self._monitor_listener:
# has path changed?
if self._monitor_listener[combined_id].path == path:
return ""
else:
self._monitor_listener[combined_id].listener.remove(combined_id)
self._check_watch(self._monitor_listener[combined_id])
del self._monitor_listener[combined_id]
# create or get the desired watch
if path not in self._monitor_watches:
if os.path.isfile(path):
watch = self._monitor_thread.add_file_watch(path)
elif os.path.isdir(path):
watch = self._monitor_thread.add_dir_watch(path)
else:
return "This kind of file can't be wachted"
watch.listener = []
self._monitor_watches[path] = watch
else:
watch = self._monitor_watches[path]
if combined_id in watch.listener:
return "This file is already watched" # this should never occur
watch.listener.append(combined_id)
self._monitor_listener[combined_id] = watch
return ""
def unwatch(self, window_id, view_id, watch_id=None):
with self._monitor_lock:
if watch_id is None:
# will perform poor with lots of watches/listener
for path, watch in self._monitor_watches.items():
for combined_id in watch.listener:
if window_id==combined_id[0] and view_id==combined_id[1]:
watch.listener.remove(combined_id)
del self._monitor_listener[combined_id]
self._check_watch(watch)
else:
combined_id = (window_id, view_id, watch_id)
if combined_id in self._monitor_listener:
self._monitor_listener[combined_id].listener.remove(combined_id)
self._check_watch(self._monitor_listener[combined_id])
del self._monitor_listener[combined_id]
return ""
def string_compare(a, b):
if a == b:
......
# Copyright (c) 2010, 2012 Luke McCarthy <luke@iogopro.co.uk>
#
# This is free software released under the MIT license.
# See COPYING file for details, or visit:
# http://www.opensource.org/licenses/mit-license.php
#
# The file is part of FSMonitor, a file-system monitoring library.
# https://github.com/shaurz/fsmonitor
import sys
import threading
import traceback
from .common import FSEvent, FSMonitorError, FSMonitorOSError
# set to None when unloaded
module_loaded = True
if sys.platform == "linux2":
from .linux import FSMonitor
elif sys.platform == "win32":
from .win32 import FSMonitor
else:
from .polling import FSMonitor
class FSMonitorThread(threading.Thread):
def __init__(self, callback=None):
threading.Thread.__init__(self)
self.monitor = FSMonitor()
self.callback = callback
self._running = True
self._events = []
self._events_lock = threading.Lock()
self.daemon = True
self.start()
def add_dir_watch(self, path, flags=FSEvent.All, user=None):
return self.monitor.add_dir_watch(path, flags=flags, user=user)
def add_file_watch(self, path, flags=FSEvent.All, user=None):
return self.monitor.add_file_watch(path, flags=flags, user=user)
def remove_watch(self, watch):
self.monitor.remove_watch(watch)
def remove_all_watches(self):
self.monitor.remove_all_watches()
with self._events_lock:
self._events = []
def run(self):
while module_loaded and self._running:
try:
events = self.monitor.read_events()
if self.callback:
for event in events:
self.callback(event)
else:
with self._events_lock:
self._events.extend(events)
except Exception:
print "Exception in FSMonitorThread:\n" + traceback.format_exc()
def stop(self):
if self.monitor.watches:
self.remove_all_watches()
self._running = False
def read_events(self):
with self._events_lock:
events = self._events
self._events = []
return events
__all__ = (
"FSMonitor",
"FSMonitorThread",
"FSMonitorError",
"FSMonitorOSError",
"FSEvent",
)
# Copyright (c) 2010 Luke McCarthy <luke@iogopro.co.uk>
#
# This is free software released under the MIT license.
# See COPYING file for details, or visit:
# http://www.opensource.org/licenses/mit-license.php
#
# The file is part of FSMonitor, a file-system monitoring library.
# https://github.com/shaurz/fsmonitor