filesystem.py 32.9 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
import mimetypes
7
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
from subprocess import call, check_output
11
import ConfigParser
12
import json
13
14
15
16
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
17
import stat
18
import subprocess
19
20
import fsmonitor
import vispa
21
import tempfile
22
23
24

try:
    import Image
Gero Müller's avatar
Gero Müller committed
25
    import ImageFilter
Martin Urban's avatar
Martin Urban committed
26

27
28
29
30
31
32
33
34
35
36
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
37

38
39
logger = logging.getLogger(__name__)

Martin Urban's avatar
Martin Urban committed
40

41
42
43
44
45
46
47
48
49
50
51
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
Martin Urban's avatar
Martin Urban committed
52

53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

asseldonk's avatar
asseldonk committed
72

Martin Urban's avatar
Martin Urban committed
73
class FileSystem(object):
74
75
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
76
77
78
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
79
        "root": "text/plain"
Marcel's avatar
Marcel committed
80
    }
81
82
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
83

84
    def __init__(self, userid, workspaceid):
85
86
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
87
        self.watchservice = WatchService()
88
89
        self._userid = userid
        self._workspaceid = workspaceid
90
        mimetypes.init()
Martin Urban's avatar
Martin Urban committed
91

92
93
    def __del__(self):
        self.close()
94

95
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
96
97
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
98
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
99
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
100
101
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
102
103
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
104
        else:
Gero Müller's avatar
Gero Müller committed
105
            os.makedirs(self.basedir, 0o700)
106
            return "Basedir now exists"
107

108
    def close(self):
109
110
        if self.watchservice:
            self.watchservice.stop()
Martin Urban's avatar
Martin Urban committed
111

murban's avatar
murban committed
112
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
113
        filepath = self.expand(filepath)
114
        mime, encoding = mimetypes.guess_type(filepath)
115
116
117
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
118
119
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
120
121
122
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
123
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
124
        path = self.expand(path)
125
126
127
128
129
130
131
132
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
133
134
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
135
        path = self.expand(path)
136
137
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
138
139
            return None
        # type correct?
140
141
142
143
144
145
146
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
147

148
149
    def get_file_count(
            self, path, window_id=None, view_id=None, watch_id=None):
150
151
152
153
154
155
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
156
157
        path = self.expand(path)
        if os.path.exists(path):
Martin Urban's avatar
Martin Urban committed
158
159
160
161
162
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
163
164
165
        else:
            return -1

166
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
167
                      filter=None, reverse=False,
168
169
170
171
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
172
173
            if self.watch(
                    path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
174
175
176
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
177
        filter = re.compile(filter) if filter else None
178
        base = self.expand(path)
Martin Urban's avatar
Martin Urban committed
179

180
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
Martin Urban's avatar
Martin Urban committed
181
182
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
183
184
        # ignore failed file info (e.g. access error)
        filelist = [i for i in filelist if 'size' in i]
185
186

        # Determine the parent
187
188
189
190
191
192
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
193
194
        if encode_json:
            return json.dumps(data)
195
196
        else:
            return data
Martin Urban's avatar
Martin Urban committed
197

Gero Müller's avatar
Gero Müller committed
198
199
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
200
201
202
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
203
204
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
205
            # dir case
Marcel's avatar
Marcel committed
206
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
207
208
209
210
211
212
213
214
215
216
217
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
218
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
219
220
221
222
223
224
225
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
226
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
227
228
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
229
230
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
231
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
232
233
234
235
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
236
237
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
238
239
240
                suggestion += '/'
            suggestions.append(suggestion)

241
        return suggestions if not encode_json else json.dumps(suggestions)
242

murban's avatar
murban committed
243
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
244
        path = self.expand(path)
murban's avatar
murban committed
245
        path = path[1:] if path.startswith(os.sep) else path
246
247
        if path == "":
            return path
murban's avatar
murban committed
248
        path = path[:-1] if path.endswith(os.sep) else path
249
250
        return path

murban's avatar
murban committed
251
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
252
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
253
254
        name = self.expand(name)

255
        # folder with the same name existent?
256
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
257
        fullpath = os.path.join(path, name)
258
259
260
        try:
            os.mkdir(fullpath)
        except Exception as e:
261
            # raise Exception("You don't have the permission to create this folder!")
262
263
            raise Exception(str(e))

murban's avatar
murban committed
264
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
265
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
266
267
        name = self.expand(name)

268
        # file with the same name existent?
269
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
270
        fullpath = os.path.join(path, name)
271
272
273
274
275
276
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

277
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
278
279
280
281
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

282
283
284
285
286
        if force == False:
            new_name = self.handle_file_name_collision(new_name, path)
        name = os.path.join(path, name)
        new_name = os.path.join(path, new_name)
        os.renames(name, new_name)
287

murban's avatar
murban committed
288
289
    def remove(self, path):
        if isinstance(path, list):
290
291
292
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
293
            path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
294
295
296
            if os.path.islink(path):
                os.unlink(path)
            elif os.path.isdir(path):
Marcel's avatar
Marcel committed
297
298
299
                shutil.rmtree(path)
            else:
                os.remove(path)
300

301
    def move(self, source, destination):
302
303
304
305
306
307
        if isinstance(source, list):
            for s in source:
                self.move(s, destination)
        else:
            source = self.expand(source)
            destination = self.expand(destination)
308
309
310
311
            name = os.path.split(source)[1]
            newname = self.handle_file_name_collision(name, destination)
            destination = os.path.join(destination, newname)
            if os.path.isdir(source):
312
                shutil.copytree(source, destination, symlinks=True)
313
314
315
316
317
                shutil.rmtree(source)
            else:
                shutil.copy2(source, destination)
                os.remove(source)

318

319
    def compress(self, paths, path, name, is_tmp=False):
Gero Müller's avatar
Gero Müller committed
320
321
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
322
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
323

324
325
326
327
328
329
330
331
332
        if is_tmp:
            fullpath = os.path.join(tempfile._get_default_tempdir(),
                                    next(tempfile._get_candidate_names()) + ".zip")
        else:
            path = path if not path.endswith(os.sep) else path[:-1]
            path = self.expand(path)
            name = name if name.endswith(".zip") else name + ".zip"
            name = self.handle_file_name_collision(name, path)
            fullpath = os.path.join(path, name)
Gero Müller's avatar
Gero Müller committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349


        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
350
                            ap = fullp[
351
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
352
353
354
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
355
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
356
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
357
                    archive.write(p, ap)
358

359
360
        return fullpath

361
    def decompress(self, path):
362
        # filepath and extract path
363
364
365
366
        path = self.expand(path)

        # "foo/bar/file.zip" -> ("foo/bar", "file")
        dst = os.path.split(os.path.splitext(path)[0])
367

368
369
370
        with ZipFile(path, "r") as archive:
            dstdir = os.path.join(dst[0], self.handle_file_name_collision(dst[1], dst[0]))
            archive.extractall(dstdir)
371

372
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
373
        # TODO
374
375
376
377
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
378
                self.paste(path, p, cut)
379
380
            return True

381
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
382
383
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
384
        fulltarget = os.path.join(path, target)
385

asseldonk's avatar
asseldonk committed
386
        if os.path.isdir(fullsrc):
387
            shutil.copytree(fullsrc, fulltarget, symlinks=True)
asseldonk's avatar
asseldonk committed
388
389
            if cut:
                shutil.rmtree(fullsrc)
390
        else:
asseldonk's avatar
asseldonk committed
391
392
393
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
394

395
    def save_file(self, path, content, force=True, binary=False,
396
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
397
        path = self.expand(path)
Marcel's avatar
Marcel committed
398
        # check if file already exists
murban's avatar
murban committed
399
        if os.path.exists(path) and not force:
400
401
402
403
404
405
406
407
408
409
410
411
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
412
413
            mtime = 0

414
415
416
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
417
        else:
418
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
419

420
421
        return json.dumps({
            "mtime": os.path.getmtime(path),
422
423
            # save is not successful, if file not writable
            "success": mtime > 0 and self.checkPermissions(path),
Martin Urban's avatar
Martin Urban committed
424
            "watch_error": watch_error,
425
            "path": path
426
        })
Martin Urban's avatar
Martin Urban committed
427

428
    def get_file(self, path, binary=False,
Martin Urban's avatar
Martin Urban committed
429
                 utf8=False, window_id=None, view_id=None, watch_id=None):
430
431
432
433
434
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
Martin Urban's avatar
Martin Urban committed
435

436
437
438
439
440
441
442
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
Martin Urban's avatar
Martin Urban committed
443
            # new: check for writing rights
444
            writable = self.checkPermissions(path)
445
            error = None
446

447
448
            mtime = os.path.getmtime(path)
        except Exception as e:
Martin Urban's avatar
Martin Urban committed
449
            mtime = 0
450
            content = ""
451
            writable = None
452
            error = str(e)
Martin Urban's avatar
Martin Urban committed
453

454
455
456
457
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
458
            "watch_error": watch_error,
Martin Urban's avatar
Martin Urban committed
459
            "writable": writable,
460
            "error": error
461
        })
462

Martin Urban's avatar
Martin Urban committed
463
464
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
465

466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
    def getfacl(self, path):
        o = check_output(["getfacl", "-p", "-t", path])
        facls = []
        for line in o.splitlines():
            if not line: continue
            if line.startswith("#"): continue
            e = line.split()
            if e[0] in ["mask", "other"]: e[1:1] = [""]
            facls.append(tuple(e))
        return facls

    def setfacl(self, path, type, name, mode, remove=False, recursive=False, default=False):
        if type not in ("user", "group", "mask", "other"):
            raise TypeError("type '%s' not in ('user', 'group', 'mask', 'other')" % type)
        if type in ("mask", "other"):
            name = ""

        arguments = ["setfacl"]

        if default:
            arguments.append("-d")

        if recursive:
            arguments.append("-R")

        if remove:
            action = "-x"
            acl = "%s:%s" % (type, name)
        else:
            action = "-m"
            acl = "%s:%s:%s" % (type, name, mode)
        arguments.append(action)
        arguments.append(acl)
        arguments.append(path)
        call(arguments, stderr=open(os.devnull, "wb"))

Gero Müller's avatar
Gero Müller committed
502
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
503
                          path=None, force=True, append=False):
504
505
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
Martin Urban's avatar
Martin Urban committed
506
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
507
508
509
510
511

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

512
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
513
514
515
516
517
518
519
520
521
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

522
523
524
525
    def get_file_content(self, path, offset=0, length=None):
        with open(self.expand(path), "rb") as f:
            f.seek(offset)
            content = f.read() if length is None else f.read(length)
Marcel's avatar
Marcel committed
526
        return content
527

Marcel's avatar
Marcel committed
528
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
529
        path = self.expand(path)
Marcel's avatar
Marcel committed
530
531
        return os.path.getmtime(path)

532
533
534
535
    def stat(self, path):
        path = self.expand(path)
        return os.stat(path)

murban's avatar
murban committed
536
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
537
        path = self.expand(path)
538
        extension = path.split(".")[-1]
539
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
540

murban's avatar
murban committed
541
    def handle_file_name_collision(self, name, path):
542
        # collision?
543
        path = self.expand(path)
murban's avatar
murban committed
544
        files = os.listdir(path)
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
562
        counter = prename.split("_copy")[-1]
563
564
565
566
567

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
568
                preprename = "_copy".join(prename.split("_copy")[:-1])
569
570
571
572
573
574
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
575
            newname = "%s_copy%d%s" % (preprename,
Gero Müller's avatar
Gero Müller committed
576
577
                                   counter,
                                   "" if extension == "" else "." + extension)
578
        else:
579
            newname = "%s_copy1%s" % (
Gero Müller's avatar
Gero Müller committed
580
                prename, "" if extension == "" else "." + extension)
581
582

        # return
murban's avatar
murban committed
583
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
584

Marcel's avatar
Marcel committed
585
586
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
587

Gero Müller's avatar
Gero Müller committed
588
    def thumbnail(self, path, width=100, height=100, sharpen=True):
589
590
591
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
592
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
593
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
594
            if sharpen:
595
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
596
            img.save(output, "JPEG")
597
598
599
600
601
602
603
604
605
606
607
608
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

609
610
    def watch(self, path, window_id, view_id, watch_id,
              pattern=None, reverse=False, hide_hidden=True):
611
612
613
614
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
Martin Urban's avatar
Martin Urban committed
615
616
617
618
619

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

620
621
        #first: remove the old watch
        self.unwatch(window_id, view_id, watch_id)
622

623
624
625
626
627
628
629
630
        self.watchservice.subscribe(
            (window_id,
             view_id,
             watch_id),
            path,
            pattern,
            reverse,
            hide_hidden)
631
        return ""
Martin Urban's avatar
Martin Urban committed
632

633
    def unwatch(self, window_id, view_id, watch_id=None):
634
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
635
636
        return ""

637
    def get_workspaceini(self, request, fail_on_missing=False):
638
        try:
639
            request_dict = json.loads(request)
640
            config = ConfigParser.ConfigParser()
641
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
Martin Urban's avatar
Martin Urban committed
642
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
643
644
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
Martin Urban's avatar
Martin Urban committed
645
                self._watch_workspaceini()
646
647
            else:
                mtime = -1
648
            if not isinstance(request_dict, dict):
649
                request_dict = dict.fromkeys(config.sections(), True)
650
651
652
653
654
655
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
656
                        data[section] = dict(config.items(section))
657
658
659
660
661
662
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
Martin Urban's avatar
Martin Urban committed
663
664
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
665
                elif fail_on_missing:
666
667
668
                    raise Exception(
                        'workspace.ini is missing the section [%s]' %
                        section)
669
            return json.dumps({
670
                "content": data,
671
672
                "success": True,
                "mtime": mtime
Martin Urban's avatar
Martin Urban committed
673
            })
674
675
        except Exception as e:
            return json.dumps({
676
677
678
                "content": "",
                "success": False,
                "error": str(e)
Martin Urban's avatar
Martin Urban committed
679
            })
680

681
    def set_workspaceini(self, request):
682
        try:
683
684
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
685
686
                raise Exception(
                    'Given values to be set in workspace.ini in wrong format')
687
            filename = self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)
688
            config = ConfigParser.ConfigParser()
689
            config.read(filename)
690
691
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
692
693
                    raise Exception(
                        'Given values to be set in workspace.ini in wrong format')
694
695
696
697
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
698
699
700
701
            filedir = os.path.dirname(filename)
            if not os.path.isdir(filedir):
                os.makedirs(filedir)
            with open(filename, 'w') as f:
702
                config.write(f)
703
            self._watch_workspaceini()
704
            return ""
705
        except Exception as e:
706
            return str(e)
asseldonk's avatar
asseldonk committed
707

708
    def _watch_workspaceini(self):
709
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
710
711
712
713
            self.watchservice.subscribe(
                (self._userid,
                 self._workspaceid),
                FileSystem.PRIVATE_WORKSPACE_CONF)
714

Martin Urban's avatar
Martin Urban committed
715

716
717
718
719
720
721
722
723
724
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
725
        self.thread.daemon = True
726
        self.thread.start()
Martin Urban's avatar
Martin Urban committed
727

728
729
    def subscribe(
            self, id, path, pattern=None, reverse=False, hide_hidden=True):
730
731
        if not path:
            return self.unsubscribe(id)
Martin Urban's avatar
Martin Urban committed
732

733
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
Martin Urban's avatar
Martin Urban committed
734

735
736
737
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
Martin Urban's avatar
Martin Urban committed
738

739
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
Martin Urban's avatar
Martin Urban committed
740

741
742
743
744
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
745
746
                    if False not in map(
                            lambda e, c: c is None or e == c, subscriber.id, id):
747
748
749
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
Martin Urban's avatar
Martin Urban committed
750

751
752
    def stop(self):
        self.run = False
Martin Urban's avatar
Martin Urban committed
753

754
755
756
757
758
759
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
Martin Urban's avatar
Martin Urban committed
760
                        if event.action_name in ['delete self', 'move self']:
761
762
763
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
Martin Urban's avatar
Martin Urban committed
764
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
765
766
767
768
769
770
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
771
772
                                    event.watch.mtime = os.path.getmtime(
                                        event.watch.path)
773
774
775
776
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
Martin Urban's avatar
Martin Urban committed
777

778
779
780
781
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
Martin Urban's avatar
Martin Urban committed
782

783
        for subscriber in self.subscribers.items():
784
            subscriber.destroy()
Martin Urban's avatar
Martin Urban committed
785

786
        self.monitor.remove_all_watches()
787
        del self.monitor
Martin Urban's avatar
Martin Urban committed
788
789
790


class WatchSubscriber(object):  # this should never be instanced manually
791
    EVENT_DELAYS = {
Martin Urban's avatar
Martin Urban committed
792
793
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
794
    }
795
796
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
Martin Urban's avatar
Martin Urban committed
797

798
799
800
801
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
802
803
804
            raise RuntimeError(
                "There is already a subscriber with this id: " +
                str(id))
805
806
807
808
809
810
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
811
        self.hide_hidden = None
812
        self.event_buffer = {}
813
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
814

815
816
817
818
819
820
821
822
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
823
        del self.subject_buffer
Martin Urban's avatar
Martin Urban committed
824

825
    def process(self, event, subject=""):
826
827
828
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
829
830
            if self.pattern and bool(
                    self.pattern.search(subject)) != self.reverse:
831
                return
Martin Urban's avatar
Martin Urban committed
832

833
834
835
836
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
Martin Urban's avatar
Martin Urban committed
837

838
839
840
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
841
                self.event_buffer[event][1] = now + \
842
                                              WatchSubscriber.EVENT_DELAYS[event][1]
843
            else:
Martin Urban's avatar
Martin Urban committed
844
845
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
846
847
848
849
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
Martin Urban's avatar
Martin Urban committed
850

851
852
853
854
855
856
857
858
859
860
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
Martin Urban's avatar
Martin Urban committed
861
        if len(self.id) == 3:  # window_id, view_id, watch_id
862
863
864
865
866
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
867
868
869
870
871
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
Martin Urban's avatar
Martin Urban committed
872
873
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
874
875
876
877
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
878
                data['mtime'] = self.watch.mtime
879
880
881
882
883
            vispa.remote.send_topic(
                "extension.%s.socket.watch" %
                self.id[1],
                window_id=self.id[0],
                data=data)
Martin Urban's avatar
Martin Urban committed
884
        elif len(self.id) == 2:  # userid, workspaceid
885
886
887
888
889
890
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
Martin Urban's avatar
Martin Urban committed
891

892
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
893
894
895
896
897
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
898
899
900
901
902
903
904
905
            old_subject_buffer = self.subject_buffer
            self.subject_buffer = {}
            for event, subjects in old_subject_buffer.items():
                new_subject_list = [
                    subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
                ]
                if len(new_subject_list):
                    self.subject_buffer[event] = new_subject_list
906
907
908
        else:
            self.pattern = None
            self.reverse = None
909
        self.hide_hidden = hide_hidden
Martin Urban's avatar
Martin Urban committed
910

911
912
913
914
915
916
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
Martin Urban's avatar
Martin Urban committed
917

918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
Martin Urban's avatar
Martin Urban committed
934

935
936
937
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
Martin Urban's avatar
Martin Urban committed
938

939
        self.subject_buffer = {}
Martin Urban's avatar
Martin Urban committed
940

941
942
    def unbind(self):
        if not self.watch:
943
            return
Martin Urban's avatar
Martin Urban committed
944