filesystem.py 30.3 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
24

25
26
27
28
29
30
31
32
33
34
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
35

36
37
logger = logging.getLogger(__name__)

38

39
40
41
42
43
44
45
46
47
48
49
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
50

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

70

71
class FileSystem(object):
72
73
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
74
75
76
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
77
        "root": "text/plain"
Marcel's avatar
Marcel committed
78
    }
79
80
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
81

82
    def __init__(self, userid, workspaceid):
83
84
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
85
        self.watchservice = WatchService()
86
87
        self._userid = userid
        self._workspaceid = workspaceid
88

89
90
    def __del__(self):
        self.close()
91

92
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
93
94
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
95
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
96
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
97
98
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
99
100
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
101
        else:
Gero Müller's avatar
Gero Müller committed
102
            os.makedirs(self.basedir, 0o700)
103
            return "Basedir now exists"
104

105
    def close(self):
106
107
        if self.watchservice:
            self.watchservice.stop()
108

murban's avatar
murban committed
109
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
110
        filepath = self.expand(filepath)
111
112
113
114
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
115
116
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
117
118
119
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
120
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
121
        path = self.expand(path)
122
123
124
125
126
127
128
129
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
130
131
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
132
        path = self.expand(path)
133
134
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
135
136
            return None
        # type correct?
137
138
139
140
141
142
143
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
144

145
146
147
148
149
150
151
    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
152
153
        path = self.expand(path)
        if os.path.exists(path):
154
155
156
157
158
159
160
            # import pydevd
            # pydevd.settrace('localhost', port=44847, stdoutToServer=False, stderrToServer=False)
            # check if reading the file is allowed
            if not self.checkPermissions(path, os.R_OK):
                return -2
            length = len(os.listdir(path))
            return length
161
162
163
        else:
            return -1

164
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
165
                      filter=None, reverse=False,
166
167
168
169
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
170
            if self.watch(path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
171
172
173
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
174
        filter = re.compile(filter) if filter else None
175
        base = self.expand(path)
176

177
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
178
179
180
                    not (hide_hidden and name.startswith('.')) and
                    (not filter or bool(filter.search(name)) == reverse)]
        filelist = [i for i in filelist if 'size' in i]  # ignore failed file info (e.g. access error)
181
182

        # Determine the parent
183
184
185
186
187
188
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
189
190
        if encode_json:
            return json.dumps(data)
191
192
        else:
            return data
193

Gero Müller's avatar
Gero Müller committed
194
195
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
196
197
198
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
199
200
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
201
            # dir case
Marcel's avatar
Marcel committed
202
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
203
204
205
206
207
208
209
210
211
212
213
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
214
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
215
216
217
218
219
220
221
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
222
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
223
224
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
225
226
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
227
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
228
229
230
231
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
232
233
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
234
235
236
                suggestion += '/'
            suggestions.append(suggestion)

237
        return suggestions if not encode_json else json.dumps(suggestions)
238

murban's avatar
murban committed
239
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
240
        path = self.expand(path)
murban's avatar
murban committed
241
        path = path[1:] if path.startswith(os.sep) else path
242
243
        if path == "":
            return path
murban's avatar
murban committed
244
        path = path[:-1] if path.endswith(os.sep) else path
245
246
        return path

murban's avatar
murban committed
247
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
248
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
249
250
        name = self.expand(name)

251
        # folder with the same name existent?
252
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
253
        fullpath = os.path.join(path, name)
254
255
256
        try:
            os.mkdir(fullpath)
        except Exception as e:
257
            # raise Exception("You don't have the permission to create this folder!")
258
259
            raise Exception(str(e))

murban's avatar
murban committed
260
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
261
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
262
263
        name = self.expand(name)

264
        # file with the same name existent?
265
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
266
        fullpath = os.path.join(path, name)
267
268
269
270
271
272
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

273
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
274
275
276
277
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

278
279
280
281
282
283
284
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
285
        except Exception as e:
286
            return str(e)
287

murban's avatar
murban committed
288
289
    def remove(self, path):
        if isinstance(path, list):
290
291
292
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
293
294
295
296
297
            path = self.expand(path)
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
298

299
300
301
302
303
304
    def move(self, source, destination):
        source = self.expand(source)
        destination = self.expand(destination)
        if os.path.isdir(destination):
            shutil.move(source, destination)

305
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
306
307
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
308
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
309
310
311

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
312

Gero Müller's avatar
Gero Müller committed
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
333
                            ap = fullp[
334
                                 len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
335
336
337
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
338
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
339
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
340
                    archive.write(p, ap)
341

342
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
343
        # TODO
344
345
346
347
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
348
                self.paste(path, p, cut)
349
350
            return True

351
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
Gero Müller's avatar
Gero Müller committed
352
353
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
354
        fulltarget = os.path.join(path, target)
355

356
357
358
359
360
361
362
363
364
365
366
367
        orig_target = fullsrc.split(os.sep)[-1]
        orig_fulltarget = os.path.join(path, orig_target)

        if fullsrc != orig_fulltarget:
            if os.path.isdir(fullsrc):
                shutil.copytree(fullsrc, fulltarget)
                if cut:
                    shutil.rmtree(fullsrc)
            else:
                shutil.copy2(fullsrc, fulltarget)
                if cut:
                    os.remove(fullsrc)
asseldonk's avatar
asseldonk committed
368

369
        else:
370
371
372
373
374
375
376
377
378
379
380
            fulltarget_temp = orig_fulltarget + "~temp"
            if os.path.isdir(fullsrc):
                shutil.copytree(fullsrc, fulltarget_temp)
                if cut:
                    shutil.rmtree(fullsrc)
                shutil.move(fulltarget_temp, orig_fulltarget)
            else:
                shutil.copy2(fullsrc, fulltarget_temp)
                if cut:
                    os.remove(fullsrc)
                shutil.move(fulltarget_temp, orig_fulltarget)
381

382
    def save_file(self, path, content, force=True, binary=False,
383
                  utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
384
        path = self.expand(path)
Marcel's avatar
Marcel committed
385
        # check if file already exists
murban's avatar
murban committed
386
        if os.path.exists(path) and not force:
387
388
389
390
391
392
393
394
395
396
397
398
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
399
400
            mtime = 0

401
402
403
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
404
        else:
405
            watch_error = ""
406

407
408
        return json.dumps({
            "mtime": os.path.getmtime(path),
409
410
            "success": mtime > 0 and self.checkPermissions(path),  # save is not successful, if file not writable
            "watch_error": watch_error,
411
            "path": path
412
        })
413

414
    def get_file(self, path, binary=False,
415
                 utf8=False, window_id=None, view_id=None, watch_id=None):
416
417
418
419
420
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
421

422
423
424
425
426
427
428
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
429
            # new: check for writing rights
430
            writable = self.checkPermissions(path)
431
            error = None
432

433
434
            mtime = os.path.getmtime(path)
        except Exception as e:
435
            mtime = 0
436
            content = ""
437
            writable = None
438
            error = str(e)
439

440
441
442
443
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
444
            "watch_error": watch_error,
445
            "writable": writable,
446
            "error": error
447
        })
448

449
450
    def checkPermissions(self, path, permission=os.W_OK):
        return os.access(path, permission)
451

Gero Müller's avatar
Gero Müller committed
452
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
453
                          path=None, force=True, append=False):
454
455
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
456
        # return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
457
458
459
460
461

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

462
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
463
464
465
466
467
468
469
470
471
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

murban's avatar
murban committed
472
    def get_file_content(self, path):
Marcel's avatar
Marcel committed
473
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
474
        f = open(path, "rb")
475
476
        content = f.read()
        f.close()
Marcel's avatar
Marcel committed
477
        return content
478

Marcel's avatar
Marcel committed
479
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
480
        path = self.expand(path)
Marcel's avatar
Marcel committed
481
482
        return os.path.getmtime(path)

murban's avatar
murban committed
483
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
484
        path = self.expand(path)
485
        extension = path.split(".")[-1]
486
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
487

murban's avatar
murban committed
488
    def handle_file_name_collision(self, name, path):
489
        # collision?
murban's avatar
murban committed
490
        files = os.listdir(path)
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
Gero Müller's avatar
Gero Müller committed
521
522
523
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
524
        else:
Gero Müller's avatar
Gero Müller committed
525
526
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)
527
528

        # return
murban's avatar
murban committed
529
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
530

Marcel's avatar
Marcel committed
531
532
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
533

Gero Müller's avatar
Gero Müller committed
534
    def thumbnail(self, path, width=100, height=100, sharpen=True):
535
536
537
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
538
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
539
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
540
            if sharpen:
541
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
542
            img.save(output, "JPEG")
543
544
545
546
547
548
549
550
551
552
553
554
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

555
    def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False, hide_hidden=True):
556
557
558
559
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
560
561
562
563
564

        # check if reading the file is allowed
        if not self.checkPermissions(path, os.R_OK):
            return "Reading the file is not allowed"

565
        self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse, hide_hidden)
566
        return ""
567

568
    def unwatch(self, window_id, view_id, watch_id=None):
569
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
570
571
        return ""

572
    def get_workspaceini(self, request, fail_on_missing=False):
573
        try:
574
            request_dict = json.loads(request)
575
            config = ConfigParser.ConfigParser()
576
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
577
                         self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
578
579
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
580
                self._watch_workspaceini()
581
582
            else:
                mtime = -1
583
            if not isinstance(request_dict, dict):
584
                request_dict = dict.fromkeys(config.sections(), True)
585
586
587
588
589
590
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
591
                        data[section] = dict(config.items(section))
592
593
594
595
596
597
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
598
599
                                raise Exception(
                                    'workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
600
                elif fail_on_missing:
asseldonk's avatar
asseldonk committed
601
                    raise Exception('workspace.ini is missing the section [%s]' % section)
602
            return json.dumps({
603
                "content": data,
604
605
                "success": True,
                "mtime": mtime
606
            })
607
608
        except Exception as e:
            return json.dumps({
609
610
611
                "content": "",
                "success": False,
                "error": str(e)
612
            })
613

614
    def set_workspaceini(self, request):
615
        try:
616
617
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
asseldonk's avatar
asseldonk committed
618
                raise Exception('Given values to be set in workspace.ini in wrong format')
619
            config = ConfigParser.ConfigParser()
620
            config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
621
622
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
asseldonk's avatar
asseldonk committed
623
                    raise Exception('Given values to be set in workspace.ini in wrong format')
624
625
626
627
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
628
629
            with open(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF), 'w') as f:
                self._watch_workspaceini()
630
631
                config.write(f)
            return ""
632
        except Exception as e:
633
            return str(e)
asseldonk's avatar
asseldonk committed
634

635
    def _watch_workspaceini(self):
636
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
asseldonk's avatar
asseldonk committed
637
            self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)
638

639

640
641
642
643
644
645
646
647
648
649
class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()
650

651
    def subscribe(self, id, path, pattern=None, reverse=False, hide_hidden=True):
652
653
        if not path:
            return self.unsubscribe(id)
654

655
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
656

657
658
659
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
660

661
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
662

663
664
665
666
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
667
                    if False not in map(lambda e, c: c is None or e == c, subscriber.id, id):
668
669
670
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
671

672
673
    def stop(self):
        self.run = False
674

675
676
677
678
679
680
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
681
                        if event.action_name in ['delete self', 'move self']:
682
683
684
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
685
                        elif event.watch.isdir and event.action_name in ['create', 'delete', 'move from', 'move to']:
686
687
688
689
690
691
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
asseldonk's avatar
asseldonk committed
692
                                    event.watch.mtime = os.path.getmtime(event.watch.path)
693
694
695
696
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
697

698
699
700
701
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
702

703
        for subscriber in self.subscribers.items():
704
            subscriber.destroy()
705

706
707
        self.monitor.remove_all_watches()
        self.monitor.close()
708
709
710


class WatchSubscriber(object):  # this should never be instanced manually
711
    EVENT_DELAYS = {
712
713
        'change': [1.0, 0.1],
        'modify': [1.0, 0.2]
714
    }
715
716
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
717

718
719
720
721
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
722
            raise RuntimeError("There is already a subscriber with this id: " + str(id))
723
724
725
726
727
728
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
729
        self.hide_hidden = None
730
        self.event_buffer = {}
731
        self.subject_buffer = {}
732

733
734
735
736
737
738
739
740
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
741
        del self.subject_buffer
742

743
    def process(self, event, subject=""):
744
745
746
747
748
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
            if self.pattern and bool(self.pattern.search(subject)) != self.reverse:
                return
749

750
751
752
753
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
754

755
756
757
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
asseldonk's avatar
asseldonk committed
758
                self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
759
            else:
760
761
                self.event_buffer[event] = [now + delay for delay in
                                            WatchSubscriber.EVENT_DELAYS[event]]  # first & last event
762
763
764
765
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
766

767
768
769
770
771
772
773
774
775
776
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
777
        if len(self.id) == 3:  # window_id, view_id, watch_id
778
779
780
781
782
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
783
784
785
786
787
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
788
789
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in
                                                 self.subject_buffer[event]]
790
791
792
793
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
794
                data['mtime'] = self.watch.mtime
asseldonk's avatar
asseldonk committed
795
            vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
796
        elif len(self.id) == 2:  # userid, workspaceid
797
798
799
800
801
802
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
803

804
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
805
806
807
808
809
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
810
811
812
            self.subject_buffer = {event: [
                subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
            ] for event, subjects in self.subject_buffer.items()}
813
814
815
        else:
            self.pattern = None
            self.reverse = None
816
        self.hide_hidden = hide_hidden
817

818
819
820
821
822
823
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
824

825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
841

842
843
844
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
845

846
        self.subject_buffer = {}
847

848
849
    def unbind(self):
        if not self.watch:
850
            return
851

852
853
854
855
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
856

857
        self.watch = None
858

859

Marcel's avatar
Marcel committed
860
861
862
863
864
865
866
867
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

868

Marcel's avatar
Marcel committed
869
870
871
872
873
874
875
876
877
def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1