filesystem.py

# -*- coding: utf-8 -*-

# imports
from StringIO import StringIO
from distutils.spawn import find_executable
from mimetypes import guess_type
from zipfile import ZipFile
from threading import Lock, Thread
from time import time
import ConfigParser
import json
import logging
import os
import re
import shutil
import stat
import subprocess
import fsmonitor
import vispa

try:
    import Image
    import ImageFilter

    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False

logger = logging.getLogger(__name__)


def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)

        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info


class FileSystem(object):
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
        "root": "text/plain"
    }
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"

    def __init__(self, userid, workspaceid):
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
        self.watchservice = WatchService()
        self._userid = userid
        self._workspaceid = workspaceid

    def __del__(self):
        self.close()

    def setup(self, basedir=None):
        if basedir is None:
            basedir = self.expand('~/')
        if not os.path.isdir(basedir):
            raise Exception("Basedir '%s' does not exist!" % basedir)
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
        else:
            os.makedirs(self.basedir, 0o700)
            return "Basedir now exists"

    def close(self):
        if self.watchservice:
            self.watchservice.stop()

    def get_mime_type(self, filepath):
        filepath = self.expand(filepath)
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

    def check_file_extension(self, path, extensions=[]):
        path = self.expand(path)
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

    def exists(self, path, type=None):
        # type may be 'f' or 'd'
        path = self.expand(path)
        # path exists physically?
        if not os.path.exists(path):
            return None
        # type correct?
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None

    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
        path = self.expand(path)
        if os.path.exists(path):
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
        else:
            return -1

    def get_file_list(self, path,
                      filter=None, reverse=False,
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
        filter = re.compile(filter) if filter else None
        base = self.expand(path)

        filelist = [get_file_info(base, name) for name in os.listdir(base) if
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
        filelist = [i for i in filelist if 'size' in i]  # ignore failed file info (e.g. access error)

        # Determine the parent
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
        if encode_json:
            return json.dumps(data)
        else:
            return data

    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
        suggestions = []
        source, filter = None, None
        # does the path exist?
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
            # dir case
            if os.path.isdir(path_expanded):
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
        # resort?
        if append_hidden:
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
        while (len(suggestions) < length or length == 0) and len(files):
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
                suggestion += '/'
            suggestions.append(suggestion)

        return suggestions if not encode_json else json.dumps(suggestions)

    def cut_slashs(self, path):
        path = self.expand(path)
        path = path[1:] if path.startswith(os.sep) else path
        if path == "":
            return path
        path = path[:-1] if path.endswith(os.sep) else path
        return path

    def create_folder(self, path, name):
        path = self.expand(path)
        name = self.expand(name)

        # folder with the same name existent?
        name = self.handle_file_name_collision(name, path)
        fullpath = os.path.join(path, name)
        try:
            os.mkdir(fullpath)
        except Exception as e:
            # raise Exception("You don't have the permission to create this folder!")
            raise Exception(str(e))

    def create_file(self, path, name):
        path = self.expand(path)
        name = self.expand(name)

        # file with the same name existent?
        name = self.handle_file_name_collision(name, path)
        fullpath = os.path.join(path, name)
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

    def rename(self, path, name, new_name, force=False):
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
        except Exception as e:
            return str(e)

    def remove(self, path):
        if isinstance(path, list):
            for p in path:
                self.remove(p)
        else:
            path = self.expand(path)
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

    def move(self, source, destination):
        source = self.expand(source)
        destination = self.expand(destination)
        if os.path.isdir(destination):
            shutil.move(source, destination)

    def compress(self, path, paths, name):
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
        paths = [self.expand(p) for p in paths]

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)

        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
                            ap = fullp[
                                 len(path):] if fullp.startswith(path) else fullp
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
                    ap = p[len(path):] if p.startswith(path) else p
                    logger.debug(p)
                    archive.write(p, ap)

    def paste(self, path, fullsrc, cut):
        # TODO
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
                self.paste(path, p, cut)
            return True

        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
        fulltarget = os.path.join(path, target)

        if os.path.isdir(fullsrc):
            shutil.copytree(fullsrc, fulltarget)
            if cut:
                shutil.rmtree(fullsrc)
        else:
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)

    def save_file(self, path, content, force=True, binary=False,
                   utf8=False, window_id=None, view_id=None, watch_id=None):
        path = self.expand(path)
        # check if file already exists
        if os.path.exists(path) and not force:
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
            mtime = 0

        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""

        return json.dumps({
            "mtime": os.path.getmtime(path),
            "success": mtime > 0 and self.checkPermissions(path),  # save is not successful, if file not writable
            "watch_error": watch_error,
            "path": path
        })

    def get_file(self, path, binary=False,
                 utf8=False, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""

        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
            # new: check for writing rights
            writable = self.checkPermissions(path)
            error = None

            mtime = os.path.getmtime(path)
        except Exception as e:
            mtime = 0
            content = ""
            writable = None
            error = str(e)

        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
            "watch_error": watch_error,
            "writable": writable,
            "error": error
        })

    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)

    def save_file_content(self, filename, content,
                          path=None, force=True, append=False):
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
        # return False, "Permission denied!"

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

        # check if file already exists
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

    def get_file_content(self, path):
        path = self.expand(path)
        f = open(path, "rb")
        content = f.read()
        f.close()
        return content

    def get_mtime(self, path):
        path = self.expand(path)
        return os.path.getmtime(path)

    def is_browser_file(self, path):
        path = self.expand(path)
        extension = path.split(".")[-1]
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS

    def handle_file_name_collision(self, name, path):
        # collision?
        files = os.listdir(path)
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
        else:
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)

        # return
        return self.handle_file_name_collision(newname, path)

    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))

    def thumbnail(self, path, width=100, height=100, sharpen=True):
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
            img = Image.open(path)
            img.thumbnail((width, height), Image.ANTIALIAS)
            if sharpen:
                img.filter(ImageFilter.SHARPEN)
            img.save(output, "JPEG")
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

    def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False, hide_hidden=True):
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

        self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse, hide_hidden)
        return ""

    def unwatch(self, window_id, view_id, watch_id=None):
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
        return ""

    def get_workspaceini(self, request, fail_on_missing=False):
        try:
            request_dict = json.loads(request)
            config = ConfigParser.ConfigParser()
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
                self._watch_workspaceini()
            else:
                mtime = -1
            if not isinstance(request_dict, dict):
                request_dict = dict.fromkeys(config.sections(), True)
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
                        data[section] = dict(config.items(section))
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
                elif fail_on_missing:
                    raise Exception('workspace.ini is missing the section [%s]' % section)
            return json.dumps({
                "content": data,
                "success": True,
                "mtime": mtime
            })
        except Exception as e:
            return json.dumps({
                "content": "",
                "success": False,
                "error": str(e)
            })

    def set_workspaceini(self, request):
        try:
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
                raise Exception('Given values to be set in workspace.ini in wrong format')
            config = ConfigParser.ConfigParser()
            config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
                    raise Exception('Given values to be set in workspace.ini in wrong format')
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
            with open(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF), 'w') as f:
                self._watch_workspaceini()
                config.write(f)
            return ""
        except Exception as e:
            return str(e)

    def _watch_workspaceini(self):
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
            self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)


class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()

    def subscribe(self, id, path, pattern=None, reverse=False, hide_hidden=True):
        if not path:
            return self.unsubscribe(id)

        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')

        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)

            self.subscribers[id].update(path, pattern, reverse, hide_hidden)

    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
                    if False not in map(lambda e, c: c is None or e == c, subscriber.id, id):
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()

    def stop(self):
        self.run = False

    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
                        if event.action_name in ['delete self', 'move self']:
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
                                    event.watch.mtime = os.path.getmtime(event.watch.path)
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)

            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)

        for subscriber in self.subscribers.items():
            subscriber.destroy()

        self.monitor.remove_all_watches()
        self.monitor.close()


class WatchSubscriber(object):  # this should never be instanced manually
    EVENT_DELAYS = {
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
    }
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25

    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
            raise RuntimeError("There is already a subscriber with this id: " + str(id))
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
        self.hide_hidden = None
        self.event_buffer = {}
        self.subject_buffer = {}

    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
        del self.subject_buffer

    def process(self, event, subject=""):
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
            if self.pattern and bool(self.pattern.search(subject)) != self.reverse:
                return

            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)

        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
                self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
            else:
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)

    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
        if len(self.id) == 3:  # window_id, view_id, watch_id
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
                data['mtime'] = self.watch.mtime
            vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
        elif len(self.id) == 2:  # userid, workspaceid
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)

    def update(self, path, pattern="", reverse=False, hide_hidden=True):
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
            self.subject_buffer = {event: [
                subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
            ] for event, subjects in self.subject_buffer.items()}
        else:
            self.pattern = None
            self.reverse = None
        self.hide_hidden = hide_hidden

    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()

        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]

        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)

        self.subject_buffer = {}

    def unbind(self):
        if not self.watch:
            return

        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)

        self.watch = None


def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1


def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1