Commit ef93f063 authored by Gero Müller's avatar Gero Müller
Browse files

implement streaming download fixes #1983

parent e89d6afe
......@@ -64,8 +64,8 @@ class AbstractController(object):
def get(self, key, *args, **kwargs):
key = key.lower()
try:
session = cherrypy.session
request = cherrypy.request
session = getattr(cherrypy, "session")
request = getattr(cherrypy, "request")
if key == "session_id":
return session.id
elif key == "user_id":
......
......@@ -3,11 +3,17 @@
# imports
import StringIO
import json
import stat
import logging
from cherrypy.lib import file_generator, cptools, httputil
import cherrypy
from vispa import MessageException
from vispa.controller import AbstractController
import cherrypy
logger = logging.getLogger(__name__)
class FSController(AbstractController):
......@@ -15,45 +21,57 @@ class FSController(AbstractController):
def __init__(self):
AbstractController.__init__(self, mount_static=False)
@staticmethod
def _stream_remote_file(fs, path):
offset = 0
buffer_size = 2 ** 20
while True:
try:
data = fs.get_file_content(path, offset, buffer_size)
except:
logger.exception("get content")
raise
l = len(data)
if l <= 0:
break
else:
offset += l
yield data
@cherrypy.expose
@cherrypy.tools.ajax(on=False)
def getfile(self, path, download=None, **kwargs):
self.release_session()
if not download or download.lower() not in ['true', '1', 'yes']:
download = False
else:
download = True
data, contenttype, _ = self.handleDownload(path)
if contenttype != None:
cherrypy.response.headers['Content-Type'] = contenttype
if download: # or not isbrowserfile:
disposition = 'attachment; filename=%s' % path.split('/')[-1]
cherrypy.response.headers['Content-Disposition'] = disposition
return data
def handleDownload(self, path):
fs = self.get('fs')
# get the content type depending on the file extension
ext = path.split('.')[-1]
mimetype = fs.get_mime_type(path)
# if mimetype is None:
# raise Exception('The file extension \'%s\ is not supported by this server' % ext)
if not fs.exists(path, 'f'):
raise MessageException('The file \'%s\' does not exist' % path)
stats = fs.stat(path)
if not stat.S_ISREG(stats.st_mode):
raise cherrypy.HTTPError(404, "Not a File!")
# Set the Last-Modified response header, so that
# modified-since validation code can work.
cherrypy.response.headers['Cache-Control'] = 'max-age=1, private, must-revalidate, no-cache'
cherrypy.response.headers['Pragma'] = 'no-cache'
t = int(fs.get_mtime(path))
cherrypy.response.headers['Last-Modified'] = httputil.HTTPDate(t)
cherrypy.response.headers['Expires'] = httputil.HTTPDate(t + 1)
headers = cherrypy.response.headers
headers[
'Cache-Control'] = 'max-age=1, private, must-revalidate, no-cache'
headers['Pragma'] = 'no-cache'
headers['Last-Modified'] = httputil.HTTPDate(stats.st_mtime)
headers['Expires'] = httputil.HTTPDate(stats.st_mtime + 1)
headers['Content-Length'] = stats.st_size
cptools.validate_since()
data = fs.get_file_content(path)
if download and download.lower() in ['true', '1', 'yes']:
disposition = 'attachment; filename=%s' % path.split('/')[-1]
headers['Content-Disposition'] = disposition
mimetype = fs.get_mime_type(path)
if mimetype is not None:
headers['Content-Type'] = mimetype
self.release()
return data, mimetype, fs.is_browser_file(path)
return FSController._stream_remote_file(fs, path)
getfile._cp_config = {'response.stream': True}
@cherrypy.expose
@cherrypy.tools.ajax(on=False)
......@@ -90,17 +108,17 @@ class FSAjaxController(AbstractController):
fs = self.get('fs')
self.release_database()
count = fs.get_file_count(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
count = fs.get_file_count(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
if count == -1:
raise MessageException("%s does not exist" % path)
# instead of raising an exception we return -2 as count to account for the fact
# that the user does not have the permission to read the path in the GUI
# elif count == -2:
# raise MessageException("You do not have rights to read %s." % path)
elif type(count) != int:
# instead of raising an exception we return -2 as count to account for
# the fact that the user does not have the permission to read the path
# in the GUI elif count == -2:
# raise MessageException("You do not have rights to read %s." % path)
elif not isinstance(count, int):
raise MessageException(count)
else:
return {"count": count}
......@@ -116,10 +134,10 @@ class FSAjaxController(AbstractController):
# get the files with the filter
return fs.get_file_list(path, filter=filefilter,
reverse=reverse, encode_json=True,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
reverse=reverse, encode_json=True,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
@cherrypy.expose
@cherrypy.tools.ajax()
......@@ -247,7 +265,7 @@ class FSAjaxController(AbstractController):
return
else:
return "File can not be opened in browser."
except Exception, e:
except Exception as e:
return str(e)
@cherrypy.expose
......@@ -260,9 +278,9 @@ class FSAjaxController(AbstractController):
length = length or 1
suggestions = fs.get_suggestions(path, length=int(
length), append_hidden=self.convert(append_hidden, bool),
encode_json=True)
encode_json=True)
return self.success(suggestions=suggestions, encode_json=True)
except Exception, e:
except Exception as e:
return self.fail(msg=str(e), encode_json=True)
@cherrypy.expose
......@@ -283,9 +301,9 @@ class FSAjaxController(AbstractController):
fs = self.get('fs')
self.release_database()
return fs.get_file(path, utf8=utf8,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
@cherrypy.expose
@cherrypy.tools.ajax()
......@@ -294,10 +312,10 @@ class FSAjaxController(AbstractController):
fs = self.get('fs')
self.release_database()
err = fs.watch(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
err = fs.watch(path,
window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
if err:
raise MessageException(err)
return {"success": not err}
......@@ -308,10 +326,10 @@ class FSAjaxController(AbstractController):
self.release_session()
fs = self.get('fs')
self.release_database()
err = fs.unwatch(window_id=self.get('window_id'),
view_id=self.get('view_id'),
watch_id=watch_id)
view_id=self.get('view_id'),
watch_id=watch_id)
if err:
raise MessageException(err)
return {"success": not err}
......@@ -331,7 +349,7 @@ class FSAjaxController(AbstractController):
self.release_session()
fs = self.get('fs')
self.release_database()
err = fs.set_workspaceini(request)
if err:
raise MessageException(err)
......
......@@ -142,7 +142,8 @@ class FileSystem(object):
return None
return target_type if target_type == type else None
def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
def get_file_count(
self, path, window_id=None, view_id=None, watch_id=None):
# inline watch
if window_id and view_id and watch_id:
if self.watch(path, window_id, view_id, watch_id):
......@@ -165,7 +166,8 @@ class FileSystem(object):
window_id=None, view_id=None, watch_id=None):
# inline watch
if window_id and view_id and watch_id:
if self.watch(path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
if self.watch(
path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
pass
# return "" # don't fail atm since it would not be caught on the client side
# actual function
......@@ -175,7 +177,8 @@ class FileSystem(object):
filelist = [get_file_info(base, name) for name in os.listdir(base) if
not (hide_hidden and name.startswith('.')) and
(not filter or bool(filter.search(name)) == reverse)]
filelist = [i for i in filelist if 'size' in i] # ignore failed file info (e.g. access error)
# ignore failed file info (e.g. access error)
filelist = [i for i in filelist if 'size' in i]
# Determine the parent
parentpath = os.path.dirname(base)
......@@ -331,7 +334,7 @@ class FileSystem(object):
paths.append(fullp)
else:
ap = fullp[
len(path):] if fullp.startswith(path) else fullp
len(path):] if fullp.startswith(path) else fullp
logger.debug(fullp)
archive.write(fullp, ap)
else:
......@@ -363,7 +366,7 @@ class FileSystem(object):
os.remove(fullsrc)
def save_file(self, path, content, force=True, binary=False,
utf8=False, window_id=None, view_id=None, watch_id=None):
utf8=False, window_id=None, view_id=None, watch_id=None):
path = self.expand(path)
# check if file already exists
if os.path.exists(path) and not force:
......@@ -389,7 +392,8 @@ class FileSystem(object):
return json.dumps({
"mtime": os.path.getmtime(path),
"success": mtime > 0 and self.checkPermissions(path), # save is not successful, if file not writable
# save is not successful, if file not writable
"success": mtime > 0 and self.checkPermissions(path),
"watch_error": watch_error,
"path": path
})
......@@ -452,10 +456,11 @@ class FileSystem(object):
return True, "File saved!"
def get_file_content(self, path):
def get_file_content(self, path, offset=0, size=None):
path = self.expand(path)
f = open(path, "rb")
content = f.read()
f.seek(offset)
content = f.read(size)
f.close()
return content
......@@ -463,6 +468,10 @@ class FileSystem(object):
path = self.expand(path)
return os.path.getmtime(path)
def stat(self, path):
path = self.expand(path)
return os.stat(path)
def is_browser_file(self, path):
path = self.expand(path)
extension = path.split(".")[-1]
......@@ -535,7 +544,8 @@ class FileSystem(object):
else:
return self.get_file_content(path)
def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False, hide_hidden=True):
def watch(self, path, window_id, view_id, watch_id,
pattern=None, reverse=False, hide_hidden=True):
# fail if there is no such fie
path = self.expand(path)
if not os.path.exists(path):
......@@ -545,7 +555,14 @@ class FileSystem(object):
if not self.checkPermissions(path, os.R_OK):
return "Reading the file is not allowed"
self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse, hide_hidden)
self.watchservice.subscribe(
(window_id,
view_id,
watch_id),
path,
pattern,
reverse,
hide_hidden)
return ""
def unwatch(self, window_id, view_id, watch_id=None):
......@@ -581,7 +598,9 @@ class FileSystem(object):
raise Exception(
'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
elif fail_on_missing:
raise Exception('workspace.ini is missing the section [%s]' % section)
raise Exception(
'workspace.ini is missing the section [%s]' %
section)
return json.dumps({
"content": data,
"success": True,
......@@ -598,12 +617,14 @@ class FileSystem(object):
try:
request_dict = json.loads(request)
if not isinstance(request_dict, dict):
raise Exception('Given values to be set in workspace.ini in wrong format')
raise Exception(
'Given values to be set in workspace.ini in wrong format')
config = ConfigParser.ConfigParser()
config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
for section, options in request_dict.iteritems():
if not isinstance(options, dict):
raise Exception('Given values to be set in workspace.ini in wrong format')
raise Exception(
'Given values to be set in workspace.ini in wrong format')
if not config.has_section(section):
config.add_section(section)
for name, value in options.iteritems():
......@@ -617,10 +638,14 @@ class FileSystem(object):
def _watch_workspaceini(self):
if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)
self.watchservice.subscribe(
(self._userid,
self._workspaceid),
FileSystem.PRIVATE_WORKSPACE_CONF)
class WatchService(object):
def __init__(self):
self.subscriber_buffer = []
self.subscribers = {}
......@@ -631,7 +656,8 @@ class WatchService(object):
self.thread = Thread(target=self._worker)
self.thread.start()
def subscribe(self, id, path, pattern=None, reverse=False, hide_hidden=True):
def subscribe(
self, id, path, pattern=None, reverse=False, hide_hidden=True):
if not path:
return self.unsubscribe(id)
......@@ -647,7 +673,8 @@ class WatchService(object):
with self.lock:
if hasattr(id, '__contains__') and None in id:
for subscriber in self.subscribers.values():
if False not in map(lambda e, c: c is None or e == c, subscriber.id, id):
if False not in map(
lambda e, c: c is None or e == c, subscriber.id, id):
subscriber.destroy()
elif id in self.subscribers:
self.subscribers[id].destroy()
......@@ -672,7 +699,8 @@ class WatchService(object):
if kind:
if not event.watch.isdir:
if os.path.exists(event.watch.path):
event.watch.mtime = os.path.getmtime(event.watch.path)
event.watch.mtime = os.path.getmtime(
event.watch.path)
else:
event.watch.mtime = -1
for subscriber in event.watch.subscribers[:]:
......@@ -702,7 +730,9 @@ class WatchSubscriber(object): # this should never be instanced manually
if not isinstance(service, WatchService):
raise TypeError("No valid WatchService instance was provided")
if id in service.subscribers:
raise RuntimeError("There is already a subscriber with this id: " + str(id))
raise RuntimeError(
"There is already a subscriber with this id: " +
str(id))
self.id = id
self.service = service
self.service.subscribers[self.id] = self
......@@ -727,7 +757,8 @@ class WatchSubscriber(object): # this should never be instanced manually
if self.watch.isdir and subject:
if self.hide_hidden and subject.startswith('.'):
return
if self.pattern and bool(self.pattern.search(subject)) != self.reverse:
if self.pattern and bool(
self.pattern.search(subject)) != self.reverse:
return
if event not in self.subject_buffer:
......@@ -738,7 +769,8 @@ class WatchSubscriber(object): # this should never be instanced manually
if event in WatchSubscriber.EVENT_DELAYS:
now = time()
if event in self.event_buffer:
self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
self.event_buffer[event][1] = now + \
WatchSubscriber.EVENT_DELAYS[event][1]
else:
self.event_buffer[event] = [now + delay for delay in
WatchSubscriber.EVENT_DELAYS[event]] # first & last event
......@@ -775,7 +807,11 @@ class WatchSubscriber(object): # this should never be instanced manually
self.subject_buffer[event] = []
else:
data['mtime'] = self.watch.mtime
vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
vispa.remote.send_topic(
"extension.%s.socket.watch" %
self.id[1],
window_id=self.id[0],
data=data)
elif len(self.id) == 2: # userid, workspaceid
vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
"workspaceId": self.id[1],
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment