filesystem.py 29.6 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
24

25
26
27
28
29
30
31
32
33
34
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
35

36
37
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
38

39
40
41
42
43
44
45
46
47
48
49
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
70

Martin Urban's avatar
Martin Urban committed
71
class FileSystem(object):
72
73
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
74
75
76
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
77
        "root": "text/plain"
Marcel's avatar
Marcel committed
78
    }
79
80
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
81

82
    def __init__(self, userid, workspaceid):
83
84
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
85
        self.watchservice = WatchService()
86
87
        self._userid = userid
        self._workspaceid = workspaceid
Martin Urban's avatar
Martin Urban committed
88

89
90
    def __del__(self):
        self.close()
91

92
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
93
94
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
95
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
96
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
97
98
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
99
100
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
101
        else:
Gero Müller's avatar
Gero Müller committed
102
            os.makedirs(self.basedir, 0o700)
103
            return "Basedir now exists"
104

105
    def close(self):
106
107
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
108

murban's avatar
murban committed
109
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
110
        filepath = self.expand(filepath)
111
112
113
114
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
115
116
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
117
118
119
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
120
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
121
        path = self.expand(path)
122
123
124
125
126
127
128
129
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
130
131
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
132
        path = self.expand(path)
133
134
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
135
136
            return None
        # type correct?
137
138
139
140
141
142
143
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
144

145
146
147
148
149
150
151
    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
152
153
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
154
155
156
157
158
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
159
160
161
        else:
            return -1

162
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
163
                      filter=None, reverse=False,
164
165
166
167
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
168
            if self.watch(path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
169
170
171
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
172
        filter = re.compile(filter) if filter else None
173
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
174

175
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
176
177
178
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
        filelist = [i for i in filelist if 'size' in i]  # ignore failed file info (e.g. access error)
179
180

        # Determine the parent
181
182
183
184
185
186
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
187
188
        if encode_json:
            return json.dumps(data)
189
190
        else:
            return data
Martin Urban's avatar
Martin Urban committed
191

Gero Müller's avatar
Gero Müller committed
192
193
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
194
195
196
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
197
198
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
199
            # dir case
Marcel's avatar
Marcel committed
200
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
201
202
203
204
205
206
207
208
209
210
211
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
212
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
213
214
215
216
217
218
219
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
220
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
221
222
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
223
224
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
225
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
226
227
228
229
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
230
231
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
232
233
234
                suggestion += '/'
            suggestions.append(suggestion)

235
        return suggestions if not encode_json else json.dumps(suggestions)
236

murban's avatar
murban committed
237
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
238
        path = self.expand(path)
murban's avatar
murban committed
239
        path = path[1:] if path.startswith(os.sep) else path
240
241
        if path == "":
            return path
murban's avatar
murban committed
242
        path = path[:-1] if path.endswith(os.sep) else path
243
244
        return path

murban's avatar
murban committed
245
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
246
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
247
248
        name = self.expand(name)

249
        # folder with the same name existent?
250
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
251
        fullpath = os.path.join(path, name)
252
253
254
        try:
            os.mkdir(fullpath)
        except Exception as e:
255
            # raise Exception("You don't have the permission to create this folder!")
256
257
            raise Exception(str(e))

murban's avatar
murban committed
258
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
259
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
260
261
        name = self.expand(name)

262
        # file with the same name existent?
263
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
264
        fullpath = os.path.join(path, name)
265
266
267
268
269
270
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

271
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
272
273
274
275
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

276
277
278
279
280
281
282
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
283
        except Exception as e:
284
            return str(e)
285

murban's avatar
murban committed
286
287
    def remove(self, path):
        if isinstance(path, list):
288
289
290
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
291
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
292
293
294
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
295
296
297
                shutil.rmtree(path)
            else:
                os.remove(path)
298

299
300
301
302
303
304
    def move(self, source, destination):
        source = self.expand(source)
        destination = self.expand(destination)
        if os.path.isdir(destination):
            shutil.move(source, destination)

305
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
306
307
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
308
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
309
310
311

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
312

Gero Müller's avatar
Gero Müller committed
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
333
                            ap = fullp[
Martin Urban's avatar
Martin Urban committed
334
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
335
336
337
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
338
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
339
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
340
                    archive.write(p, ap)
341

342
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
343
        # TODO
344
345
346
347
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
348
                self.paste(path, p, cut)
349
350
            return True

351
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
352
353
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
354
        fulltarget = os.path.join(path, target)
355

asseldonk's avatar
asseldonk committed
356
357
358
359
        if os.path.isdir(fullsrc):
            shutil.copytree(fullsrc, fulltarget)
            if cut:
                shutil.rmtree(fullsrc)
360
        else:
asseldonk's avatar
asseldonk committed
361
362
363
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
364

365
    def save_file(self, path, content, force=True, binary=False,
asseldonk's avatar
asseldonk committed
366
                   utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
367
        path = self.expand(path)
Marcel's avatar
Marcel committed
368
        # check if file already exists
murban's avatar
murban committed
369
        if os.path.exists(path) and not force:
370
371
372
373
374
375
376
377
378
379
380
381
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
382
383
            mtime = 0

384
385
386
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
387
        else:
388
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
389

390
391
        return json.dumps({
            "mtime": os.path.getmtime(path),
Martin Urban's avatar
Martin Urban committed
392
393
            "success": mtime > 0 and self.checkPermissions(path),  # save is not successful, if file not writable
            "watch_error": watch_error,
394
            "path": path
395
        })
Martin Urban's avatar
Martin Urban committed
396

397
    def get_file(self, path, binary=False,
Martin Urban's avatar
Martin Urban committed
398
                 utf8=False, window_id=None, view_id=None, watch_id=None):
399
400
401
402
403
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
404

405
406
407
408
409
410
411
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
Martin Urban's avatar
Martin Urban committed
412
            # new: check for writing rights
413
            writable = self.checkPermissions(path)
414
            error = None
415

416
417
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
418
            mtime = 0
419
            content = ""
420
            writable = None
421
            error = str(e)
Martin Urban's avatar
Martin Urban committed
422

423
424
425
426
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
427
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
428
            "writable": writable,
429
            "error": error
430
        })
431

Martin Urban's avatar
Martin Urban committed
432
433
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
434

Gero Müller's avatar
Gero Müller committed
435
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
436
                          path=None, force=True, append=False):
437
438
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
439
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
440
441
442
443
444

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

445
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
446
447
448
449
450
451
452
453
454
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

murban's avatar
murban committed
455
    def get_file_content(self, path):
Marcel's avatar
Marcel committed
456
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
457
        f = open(path, "rb")
458
459
        content = f.read()
        f.close()
Marcel's avatar
Marcel committed
460
        return content
461

Marcel's avatar
Marcel committed
462
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
463
        path = self.expand(path)
Marcel's avatar
Marcel committed
464
465
        return os.path.getmtime(path)

murban's avatar
murban committed
466
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
467
        path = self.expand(path)
468
        extension = path.split(".")[-1]
469
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
470

murban's avatar
murban committed
471
    def handle_file_name_collision(self, name, path):
472
        # collision?
murban's avatar
murban committed
473
        files = os.listdir(path)
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
Gero Müller's avatar
Gero Müller committed
504
505
506
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
507
        else:
Gero Müller's avatar
Gero Müller committed
508
509
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)
510
511

        # return
murban's avatar
murban committed
512
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
513

Marcel's avatar
Marcel committed
514
515
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
516

Gero Müller's avatar
Gero Müller committed
517
    def thumbnail(self, path, width=100, height=100, sharpen=True):
518
519
520
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
521
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
522
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
523
            if sharpen:
524
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
525
            img.save(output, "JPEG")
526
527
528
529
530
531
532
533
534
535
536
537
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

538
    def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False, hide_hidden=True):
539
540
541
542
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
543
544
545
546
547

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

548
        self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse, hide_hidden)
549
        return ""
Martin Urban's avatar
Martin Urban committed
550

551
    def unwatch(self, window_id, view_id, watch_id=None):
552
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
553
554
        return ""

555
    def get_workspaceini(self, request, fail_on_missing=False):
556
        try:
557
            request_dict = json.loads(request)
558
            config = ConfigParser.ConfigParser()
559
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
560
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
561
562
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
563
                self._watch_workspaceini()
564
565
            else:
                mtime = -1
566
            if not isinstance(request_dict, dict):
567
                request_dict = dict.fromkeys(config.sections(), True)
568
569
570
571
572
573
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
574
                        data[section] = dict(config.items(section))
575
576
577
578
579
580
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
581
582
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
583
                elif fail_on_missing:
asseldonk's avatar
asseldonk committed
584
                    raise Exception('workspace.ini is missing the section [%s]' % section)
585
            return json.dumps({
586
                "content": data,
587
588
                "success": True,
                "mtime": mtime
Martin Urban's avatar
Martin Urban committed
589
            })
590
591
        except Exception as e:
            return json.dumps({
592
593
594
                "content": "",
                "success": False,
                "error": str(e)
Martin Urban's avatar
Martin Urban committed
595
            })
596

597
    def set_workspaceini(self, request):
598
        try:
599
600
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
asseldonk's avatar
asseldonk committed
601
                raise Exception('Given values to be set in workspace.ini in wrong format')
602
            config = ConfigParser.ConfigParser()
603
            config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
604
605
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
asseldonk's avatar
asseldonk committed
606
                    raise Exception('Given values to be set in workspace.ini in wrong format')
607
608
609
610
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
611
612
            with open(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF), 'w') as f:
                self._watch_workspaceini()
613
614
                config.write(f)
            return ""
615
        except Exception as e:
616
            return str(e)
asseldonk's avatar
asseldonk committed
617

618
    def _watch_workspaceini(self):
619
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
asseldonk's avatar
asseldonk committed
620
            self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)
621

Martin Urban's avatar
Martin Urban committed
622

623
624
625
626
627
628
629
630
631
632
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
633

634
    def subscribe(self, id, path, pattern=None, reverse=False, hide_hidden=True):
635
636
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
637

638
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
639

640
641
642
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
643

644
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
645

646
647
648
649
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
Martin Urban's avatar
Martin Urban committed
650
                    if False not in map(lambda e, c: c is None or e == c, subscriber.id, id):
651
652
653
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
654

655
656
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
657

658
659
660
661
662
663
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
664
                        if event.action_name in ['delete self', 'move self']:
665
666
667
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
668
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
669
670
671
672
673
674
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
asseldonk's avatar
asseldonk committed
675
                                    event.watch.mtime = os.path.getmtime(event.watch.path)
676
677
678
679
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
680

681
682
683
684
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
685

686
        for subscriber in self.subscribers.items():
687
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
688

689
690
        self.monitor.remove_all_watches()
        self.monitor.close()
Martin Urban's avatar
Martin Urban committed
691
692
693


class WatchSubscriber(object):  # this should never be instanced manually
694
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
695
696
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
697
    }
698
699
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
700

701
702
703
704
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
Martin Urban's avatar
Martin Urban committed
705
            raise RuntimeError("There is already a subscriber with this id: " + str(id))
706
707
708
709
710
711
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
712
        self.hide_hidden = None
713
        self.event_buffer = {}
714
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
715

716
717
718
719
720
721
722
723
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
724
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
725

726
    def process(self, event, subject=""):
727
728
729
730
731
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
            if self.pattern and bool(self.pattern.search(subject)) != self.reverse:
                return
Martin Urban's avatar
Martin Urban committed
732

733
734
735
736
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
737

738
739
740
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
asseldonk's avatar
asseldonk committed
741
                self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
742
            else:
Martin Urban's avatar
Martin Urban committed
743
744
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
745
746
747
748
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
Martin Urban's avatar
Martin Urban committed
749

750
751
752
753
754
755
756
757
758
759
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
Martin Urban's avatar
Martin Urban committed
760
        if len(self.id) == 3:  # window_id, view_id, watch_id
761
762
763
764
765
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
766
767
768
769
770
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
Martin Urban's avatar
Martin Urban committed
771
772
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
773
774
775
776
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
777
                data['mtime'] = self.watch.mtime
asseldonk's avatar
asseldonk committed
778
            vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
Martin Urban's avatar
Martin Urban committed
779
        elif len(self.id) == 2:  # userid, workspaceid
780
781
782
783
784
785
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
Martin Urban's avatar
Martin Urban committed
786

787
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
788
789
790
791
792
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
Martin Urban's avatar
Martin Urban committed
793
794
795
            self.subject_buffer = {event: [
                subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
            ] for event, subjects in self.subject_buffer.items()}
796
797
798
        else:
            self.pattern = None
            self.reverse = None
799
        self.hide_hidden = hide_hidden
Martin Urban's avatar
Martin Urban committed
800

801
802
803
804
805
806
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
Martin Urban's avatar
Martin Urban committed
807

808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
Martin Urban's avatar
Martin Urban committed
824

825
826
827
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
Martin Urban's avatar
Martin Urban committed
828

829
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
830

831
832
    def unbind(self):
        if not self.watch:
833
            return
Martin Urban's avatar
Martin Urban committed
834

835
836
837
838
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
Martin Urban's avatar
Martin Urban committed
839

840
        self.watch = None
841

Martin Urban's avatar
Martin Urban committed
842

Marcel's avatar
Marcel committed
843
844
845
846
847
848
849
850
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

Martin Urban's avatar
Martin Urban committed
851

Marcel's avatar
Marcel committed
852
853
854
855
856
857
858
859
860
def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1