filesystem.py 29.3 KB
Newer Older
Marcel's avatar
Marcel committed
1
# -*- coding: utf-8 -*-
2

Marcel's avatar
Marcel committed
3
# imports
4
5
from StringIO import StringIO
from distutils.spawn import find_executable
6
7
from mimetypes import guess_type
from zipfile import ZipFile
8
9
from threading import Lock, Thread
from time import time
10
import ConfigParser
11
import json
12
13
14
15
import logging
import os
import re
import shutil
Gero Müller's avatar
Gero Müller committed
16
import stat
17
import subprocess
18
19
import fsmonitor
import vispa
20
21
22

try:
    import Image
Gero Müller's avatar
Gero Müller committed
23
    import ImageFilter
24
25
26
27
28
29
30
31
32
33
    HAVE_PIL = True
except:
    HAVE_PIL = False

if not HAVE_PIL:
    convert_executable = find_executable('convert')
    if convert_executable and os.path.isfile(convert_executable):
        HAVE_CONVERT = True
    else:
        HAVE_CONVERT = False
34

35
36
logger = logging.getLogger(__name__)

37
38
39
40
41
42
43
44
45
46
47
def get_file_info(base, name):
    root, ext = os.path.splitext(name)
    info = {
        'name': name,
        'root': root,
        'ext': ext
    }

    try:
        fullpath = os.path.join(base, name)
        stats = os.lstat(fullpath)
asseldonk's avatar
asseldonk committed
48
        
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
        if stat.S_ISLNK(stats.st_mode):
            realfullpath = os.path.realpath(fullpath)
            info.update({
                'symlink': True,
                'realpath': realfullpath
            })
            if os.path.exists(realfullpath):
                stats = os.stat(realfullpath)

        info.update({
            'size': stats.st_size,
            'mtime': stats.st_mtime,
            'type': 'd' if stat.S_ISDIR(stats.st_mode) else 'f'
        })
    except:
        pass

    return info

68
class FileSystem(object):
asseldonk's avatar
asseldonk committed
69

70
71
    FILE_EXTENSIONS = ["png", "jpg", "jpeg", "bmp", "ps", "eps", "pdf",
                       "txt", "xml", "py", "c", "cpp", "root", "pxlio"]
Marcel's avatar
Marcel committed
72
73
74
    BROWSER_EXTENSIONS = ["png", "jpg", "jpeg", "bmp"]
    ADDITIONAL_MIMES = {
        "pxlio": "text/plain",
75
        "root": "text/plain"
Marcel's avatar
Marcel committed
76
    }
77
78
    PRIVATE_WORKSPACE_CONF = "~/.vispa/workspace.ini"
    GLOBAL_WORKSPACE_CONF = "/etc/vispa/workspace.ini"
79

80
    def __init__(self, userid, workspaceid):
81
82
        # allowed extensions
        self.allowed_extensions = FileSystem.FILE_EXTENSIONS
83
        self.watchservice = WatchService()
84
85
        self._userid = userid
        self._workspaceid = workspaceid
asseldonk's avatar
asseldonk committed
86
    
87
88
    def __del__(self):
        self.close()
89

90
    def setup(self, basedir=None):
Marcel's avatar
Marcel committed
91
92
        if basedir is None:
            basedir = self.expand('~/')
murban's avatar
murban committed
93
        if not os.path.isdir(basedir):
Marcel's avatar
Marcel committed
94
            raise Exception("Basedir '%s' does not exist!" % basedir)
murban's avatar
murban committed
95
96
        # the basedir
        self.basedir = os.path.join(basedir, ".vispa")
97
98
        if os.path.isdir(self.basedir):
            return "Basedir already exists"
99
        else:
Gero Müller's avatar
Gero Müller committed
100
            os.makedirs(self.basedir, 0o700)
101
            return "Basedir now exists"
102

103
    def close(self):
104
105
        if self.watchservice:
            self.watchservice.stop()
asseldonk's avatar
asseldonk committed
106
    
murban's avatar
murban committed
107
    def get_mime_type(self, filepath):
Marcel's avatar
Marcel committed
108
        filepath = self.expand(filepath)
109
110
111
112
        mime, encoding = guess_type(filepath)
        if mime is not None:
            return mime
        ext = filepath.split(".")[-1]
Gero Müller's avatar
Gero Müller committed
113
114
        if ext is not None and ext != "" and ext.lower(
        ) in FileSystem.ADDITIONAL_MIMES.keys():
115
116
117
            return FileSystem.ADDITIONAL_MIMES[ext]
        return None

murban's avatar
murban committed
118
    def check_file_extension(self, path, extensions=[]):
Marcel's avatar
Marcel committed
119
        path = self.expand(path)
120
121
122
123
124
125
126
127
        if (len(extensions) == 0):
            return True
        for elem in extensions:
            elem = elem if elem.startswith(".") else "." + elem
            if path.lower().endswith(elem.lower()):
                return True
        return False

Marcel's avatar
Marcel committed
128
129
    def exists(self, path, type=None):
        # type may be 'f' or 'd'
Marcel's avatar
Marcel committed
130
        path = self.expand(path)
131
132
        # path exists physically?
        if not os.path.exists(path):
Marcel's avatar
Marcel committed
133
134
            return None
        # type correct?
135
136
137
138
139
140
141
        target_type = 'd' if os.path.isdir(path) else 'f'
        if not type:
            return target_type
        type = type.lower()
        if type not in ['f', 'd']:
            return None
        return target_type if target_type == type else None
142

143
144
145
146
147
148
149
    def get_file_count(self, path, window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
            if self.watch(path, window_id, view_id, watch_id):
                pass
                # return -2 # don't fail atm since it would emit the wrong error message
        # actual function
150
151
        path = self.expand(path)
        if os.path.exists(path):
asseldonk's avatar
asseldonk committed
152
            return len(os.listdir(path))
153
154
155
        else:
            return -1

156
    def get_file_list(self, path,
Gero Müller's avatar
Gero Müller committed
157
                      filter=None, reverse=False,
158
159
160
161
                      hide_hidden=True, encode_json=True,
                      window_id=None, view_id=None, watch_id=None):
        # inline watch
        if window_id and view_id and watch_id:
162
            if self.watch(path, window_id, view_id, watch_id, filter, reverse, hide_hidden):
163
164
165
                pass
                # return "" # don't fail atm since it would not be caught on the client side
        # actual function
166
        filter = re.compile(filter) if filter else None
167
        base = self.expand(path)
asseldonk's avatar
asseldonk committed
168
        
169
        filelist = [get_file_info(base, name) for name in os.listdir(base) if
asseldonk's avatar
asseldonk committed
170
171
172
            not (hide_hidden and name.startswith('.')) and 
            (not filter or bool(filter.search(name)) == reverse)]
        filelist = [i for i in filelist if 'size' in i] # ignore failed file info (e.g. access error)
173
174

        # Determine the parent
175
176
177
178
179
180
        parentpath = os.path.dirname(base)
        data = {
            'filelist': filelist,
            'parentpath': parentpath,
            'path': base
        }
181
182
        if encode_json:
            return json.dumps(data)
183
184
        else:
            return data
asseldonk's avatar
asseldonk committed
185
    
Gero Müller's avatar
Gero Müller committed
186
187
    def get_suggestions(
            self, path, length=1, append_hidden=True, encode_json=True):
Marcel's avatar
Marcel committed
188
189
190
        suggestions = []
        source, filter = None, None
        # does the path exist?
Marcel's avatar
Marcel committed
191
192
        path_expanded = self.expand(path)
        if os.path.exists(path_expanded):
Marcel's avatar
Marcel committed
193
            # dir case
Marcel's avatar
Marcel committed
194
            if os.path.isdir(path_expanded):
Marcel's avatar
Marcel committed
195
196
197
198
199
200
201
202
203
204
205
                if path.endswith('/'):
                    source = path
                else:
                    suggestions.append(path + os.sep)
                    return suggestions
            # file case
            else:
                return suggestions
        else:
            # try to estimate source and filter
            head, tail = os.path.split(path)
Marcel's avatar
Marcel committed
206
            if os.path.isdir(os.path.expanduser(os.path.expandvars(head))):
Marcel's avatar
Marcel committed
207
208
209
210
211
212
213
                source = head
                filter = tail

        # return empty suggestions when source is not set
        if not source:
            return suggestions

Marcel's avatar
Marcel committed
214
        files = os.listdir(os.path.expanduser(os.path.expandvars(source)))
Marcel's avatar
Marcel committed
215
216
        # resort?
        if append_hidden:
Gero Müller's avatar
Gero Müller committed
217
218
            files = sorted(
                map(lambda f: str(f), files), cmp=file_compare, key=str.lower)
219
        while (len(suggestions) < length or length == 0) and len(files):
Marcel's avatar
Marcel committed
220
221
222
223
            file = files.pop(0)
            if filter and not file.startswith(filter):
                continue
            suggestion = os.path.join(source, file)
Gero Müller's avatar
Gero Müller committed
224
225
            if not suggestion.endswith(
                    '/') and os.path.isdir(os.path.expanduser(os.path.expandvars(suggestion))):
Marcel's avatar
Marcel committed
226
227
228
                suggestion += '/'
            suggestions.append(suggestion)

229
        return suggestions if not encode_json else json.dumps(suggestions)
230

murban's avatar
murban committed
231
    def cut_slashs(self, path):
Marcel's avatar
Marcel committed
232
        path = self.expand(path)
murban's avatar
murban committed
233
        path = path[1:] if path.startswith(os.sep) else path
234
235
        if path == "":
            return path
murban's avatar
murban committed
236
        path = path[:-1] if path.endswith(os.sep) else path
237
238
        return path

murban's avatar
murban committed
239
    def create_folder(self, path, name):
Marcel's avatar
Marcel committed
240
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
241
242
        name = self.expand(name)

243
        # folder with the same name existent?
244
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
245
        fullpath = os.path.join(path, name)
246
247
248
        try:
            os.mkdir(fullpath)
        except Exception as e:
249
            # raise Exception("You don't have the permission to create this folder!")
250
251
            raise Exception(str(e))

murban's avatar
murban committed
252
    def create_file(self, path, name):
Marcel's avatar
Marcel committed
253
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
254
255
        name = self.expand(name)

256
        # file with the same name existent?
257
        name = self.handle_file_name_collision(name, path)
Marcel's avatar
Marcel committed
258
        fullpath = os.path.join(path, name)
259
260
261
262
263
264
        try:
            f = file(fullpath, "w")
            f.close()
        except Exception as e:
            raise Exception(str(e))

265
    def rename(self, path, name, new_name, force=False):
Gero Müller's avatar
Gero Müller committed
266
267
268
269
        path = self.expand(path)
        name = self.expand(name)
        new_name = self.expand(new_name)

270
271
272
273
274
275
276
        try:
            if force == False:
                new_name = self.handle_file_name_collision(new_name, path)
            name = os.path.join(path, name)
            new_name = os.path.join(path, new_name)
            os.renames(name, new_name)
            return
Gero Müller's avatar
Gero Müller committed
277
        except Exception as e:
278
            return str(e)
279

murban's avatar
murban committed
280
281
    def remove(self, path):
        if isinstance(path, list):
282
283
284
            for p in path:
                self.remove(p)
        else:
Marcel's avatar
Marcel committed
285
286
287
288
289
            path = self.expand(path)
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)
290

291
292
293
294
295
296
    def move(self, source, destination):
        source = self.expand(source)
        destination = self.expand(destination)
        if os.path.isdir(destination):
            shutil.move(source, destination)

297
    def compress(self, path, paths, name):
Gero Müller's avatar
Gero Müller committed
298
299
        # paths has to be a list of strings
        paths = paths if isinstance(paths, (list, tuple)) else [paths]
Gero Müller's avatar
Gero Müller committed
300
        paths = [self.expand(p) for p in paths]
Gero Müller's avatar
Gero Müller committed
301
302
303

        path = path if not path.endswith(os.sep) else path[:-1]
        path = self.expand(path)
Martin Urban's avatar
Martin Urban committed
304

Gero Müller's avatar
Gero Müller committed
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
        name = name if name.endswith(".zip") else name + ".zip"
        name = self.handle_file_name_collision(name, path)

        fullpath = os.path.join(path, name)

        with ZipFile(fullpath, "w") as archive:
            i = 0
            while i < len(paths):
                if not paths[i]:
                    i += 1
                    continue
                p = self.expand(paths[i])
                i += 1

                if os.path.isdir(p):
                    for elem in os.listdir(p):
                        fullp = os.path.join(p, elem)
                        if os.path.isdir(fullp):
                            paths.append(fullp)
                        else:
Gero Müller's avatar
Gero Müller committed
325
                            ap = fullp[
asseldonk's avatar
asseldonk committed
326
                                len(path):] if fullp.startswith(path) else fullp
Gero Müller's avatar
Gero Müller committed
327
328
329
                            logger.debug(fullp)
                            archive.write(fullp, ap)
                else:
Martin Urban's avatar
Martin Urban committed
330
                    ap = p[len(path):] if p.startswith(path) else p
Gero Müller's avatar
Gero Müller committed
331
                    logger.debug(p)
Martin Urban's avatar
Martin Urban committed
332
                    archive.write(p, ap)
333

334
    def paste(self, path, fullsrc, cut):
Marcel's avatar
Marcel committed
335
        # TODO
336
337
338
339
        path = self.expand(path)
        if isinstance(fullsrc, (list, tuple)):
            for p in fullsrc:
                p = self.expand(p)
murban's avatar
murban committed
340
                self.paste(path, p, cut)
341
342
            return True

343
        # fulltarget = os.path.join(path, fullsrc.split(os.sep)[-1])
asseldonk's avatar
asseldonk committed
344
        logger.error("")
Gero Müller's avatar
Gero Müller committed
345
346
        target = self.handle_file_name_collision(
            fullsrc.split(os.sep)[-1], path)
347
        fulltarget = os.path.join(path, target)
348

asseldonk's avatar
asseldonk committed
349
350
351
352
        if os.path.isdir(fullsrc):
            shutil.copytree(fullsrc, fulltarget)
            if cut:
                shutil.rmtree(fullsrc)
353
        else:
asseldonk's avatar
asseldonk committed
354
355
356
            shutil.copy2(fullsrc, fulltarget)
            if cut:
                os.remove(fullsrc)
357

358
    def save_file(self, path, content, force=True, binary=False,
asseldonk's avatar
asseldonk committed
359
                   utf8=False, window_id=None, view_id=None, watch_id=None):
Marcel's avatar
Marcel committed
360
        path = self.expand(path)
Marcel's avatar
Marcel committed
361
        # check if file already exists
murban's avatar
murban committed
362
        if os.path.exists(path) and not force:
363
364
365
366
367
368
369
370
371
372
373
374
            return json.dumps({
                "mtime": 0,
                "success": False,
                "watch_error": ""
            })
        if utf8:
            content = content.encode('utf8')
        try:
            with open(path, "wb" if binary else "w") as f:
                f.write(content)
            mtime = os.path.getmtime(path)
        except Exception as e:
asseldonk's avatar
asseldonk committed
375
376
            mtime  = 0
        
377
378
379
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
380
        else:
381
            watch_error = ""
asseldonk's avatar
asseldonk committed
382
        
383
384
        return json.dumps({
            "mtime": os.path.getmtime(path),
asseldonk's avatar
asseldonk committed
385
386
            "success": mtime > 0 and self.checkPermissions(path),   #save is not successful, if file not writable    
            "watch_error": watch_error, 
387
            "path": path
388
        })
asseldonk's avatar
asseldonk committed
389
    
390
    def get_file(self, path, binary=False,
asseldonk's avatar
asseldonk committed
391
                  utf8=False, window_id=None, view_id=None, watch_id=None):
392
393
394
395
396
        # inline watch
        if window_id and view_id and watch_id:
            watch_error = self.watch(path, window_id, view_id, watch_id)
        else:
            watch_error = ""
asseldonk's avatar
asseldonk committed
397
        
398
399
400
401
402
403
404
        # actual function
        path = self.expand(path)
        try:
            with open(path, "rb" if binary else "r") as f:
                content = f.read()
            if utf8:
                content = content.decode('utf8')
asseldonk's avatar
asseldonk committed
405
            #new: check for writing rights
406
            writable = self.checkPermissions(path)
407
            error = None
408

409
410
            mtime = os.path.getmtime(path)
        except Exception as e:
asseldonk's avatar
asseldonk committed
411
            mtime  = 0
412
            content = ""
413
            writable = None
414
            error = str(e)
asseldonk's avatar
asseldonk committed
415
        
416
417
418
419
        return json.dumps({
            "content": content,
            "mtime": mtime,
            "success": mtime > 0,
420
            "watch_error": watch_error,
asseldonk's avatar
asseldonk committed
421
            "writable": writable, 
422
            "error": error
423
        })
424

asseldonk's avatar
asseldonk committed
425
426
    def checkPermissions(self, path):
        return os.access(path, os.W_OK)
427

Gero Müller's avatar
Gero Müller committed
428
    def save_file_content(self, filename, content,
Gero Müller's avatar
Gero Müller committed
429
                          path=None, force=True, append=False):
430
431
        # check write permissions
        # if not self.checkPermission(username, vPath, 'w'):
asseldonk's avatar
asseldonk committed
432
        #    return False, "Permission denied!"
Gero Müller's avatar
Gero Müller committed
433
434
435
436
437

        if path:
            filename = os.path.join(path, filename)
        filename = self.expand(filename)

438
        # check if file already exists
Gero Müller's avatar
Gero Müller committed
439
440
441
442
443
444
445
446
447
        if os.path.exists(filename) and not force:
            return False, "The file '%s' already exists!" % filename

        out = open(filename, "ab+" if append else "wb")
        out.write(content)
        out.close()

        return True, "File saved!"

murban's avatar
murban committed
448
    def get_file_content(self, path):
Marcel's avatar
Marcel committed
449
        path = self.expand(path)
Gero Müller's avatar
Gero Müller committed
450
        f = open(path, "rb")
451
452
        content = f.read()
        f.close()
Marcel's avatar
Marcel committed
453
        return content
454

Marcel's avatar
Marcel committed
455
    def get_mtime(self, path):
Marcel's avatar
Marcel committed
456
        path = self.expand(path)
Marcel's avatar
Marcel committed
457
458
        return os.path.getmtime(path)

murban's avatar
murban committed
459
    def is_browser_file(self, path):
Marcel's avatar
Marcel committed
460
        path = self.expand(path)
461
        extension = path.split(".")[-1]
462
        return extension.lower() in FileSystem.BROWSER_EXTENSIONS
463

murban's avatar
murban committed
464
    def handle_file_name_collision(self, name, path):
465
        # collision?
murban's avatar
murban committed
466
        files = os.listdir(path)
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
        if name not in files:
            return name

        # when this line is reached, there is a collision!

        # cut the file extension
        extension = name.split(".")[-1]
        prename = None
        if name == extension:
            extension = ""
            prename = name
        else:
            prename = name.split("." + extension)[0]

        # has the name already a counter at its end?
        hasCounter = False
        preprename = None
        counter = prename.split("_")[-1]

        if counter != prename:
            try:
                counter = int(counter)
                hasCounter = True
                preprename = "_".join(prename.split("_")[:-1])
            except:
                pass

        if hasCounter:
            # increment and try again
            counter += 1
Gero Müller's avatar
Gero Müller committed
497
498
499
            newname = "%s_%d%s" % (preprename,
                                   counter,
                                   "" if extension == "" else "." + extension)
500
        else:
Gero Müller's avatar
Gero Müller committed
501
502
            newname = "%s_1%s" % (
                prename, "" if extension == "" else "." + extension)
503
504

        # return
murban's avatar
murban committed
505
        return self.handle_file_name_collision(newname, path)
Marcel's avatar
Marcel committed
506

Marcel's avatar
Marcel committed
507
508
    def expand(self, path):
        return os.path.expanduser(os.path.expandvars(path))
Marcel's avatar
Marcel committed
509

Gero Müller's avatar
Gero Müller committed
510
    def thumbnail(self, path, width=100, height=100, sharpen=True):
511
512
513
        path = self.expand(path)
        if HAVE_PIL:
            output = StringIO()
Gero Müller's avatar
Gero Müller committed
514
            img = Image.open(path)
Gero Müller's avatar
Gero Müller committed
515
            img.thumbnail((width, height), Image.ANTIALIAS)
Gero Müller's avatar
Gero Müller committed
516
            if sharpen:
517
                img.filter(ImageFilter.SHARPEN)
Gero Müller's avatar
Gero Müller committed
518
            img.save(output, "JPEG")
519
520
521
522
523
524
525
526
527
528
529
530
            contents = output.getvalue()
            output.close()
            return contents
        elif HAVE_CONVERT:
            cmd = ['convert', path,
                   '-thumbnail', '%dx%d' % (width, height),
                   'jpeg:-']
            p = subprocess.Popen(cmd, stdin=None, stdout=subprocess.PIPE)
            return p.communicate()[0]
        else:
            return self.get_file_content(path)

531
    def watch(self, path, window_id, view_id, watch_id, pattern=None, reverse=False, hide_hidden=True):
532
533
534
535
        # fail if there is no such fie
        path = self.expand(path)
        if not os.path.exists(path):
            return "The file does not exist"
asseldonk's avatar
asseldonk committed
536
        
537
        self.watchservice.subscribe((window_id, view_id, watch_id), path, pattern, reverse, hide_hidden)
538
        return ""
asseldonk's avatar
asseldonk committed
539
        
540
    def unwatch(self, window_id, view_id, watch_id=None):
541
        self.watchservice.unsubscribe((window_id, view_id, watch_id))
542
543
        return ""

544
    def get_workspaceini(self, request, fail_on_missing=False):
545
        try:
546
            request_dict = json.loads(request)
547
            config = ConfigParser.ConfigParser()
548
            config.read([FileSystem.GLOBAL_WORKSPACE_CONF,
asseldonk's avatar
asseldonk committed
549
                    self.expand(FileSystem.PRIVATE_WORKSPACE_CONF)])
550
551
            if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF):
                mtime = self.get_mtime(FileSystem.PRIVATE_WORKSPACE_CONF)
asseldonk's avatar
asseldonk committed
552
                self._watch_workspaceini() 
553
554
            else:
                mtime = -1
555
            if not isinstance(request_dict, dict):
556
                request_dict = dict.fromkeys(config.sections(), True)
557
558
559
560
561
562
            data = {}
            for section, name_list in request_dict.iteritems():
                if config.has_section(section):
                    if isinstance(name_list, basestring):
                        name_list = [name_list]
                    if not isinstance(name_list, list):
563
                        data[section] = dict(config.items(section))
564
565
566
567
568
569
                    else:
                        data[section] = {}
                        for name in name_list:
                            if config.has_option(section, name):
                                data[section][name] = config.get(section, name)
                            elif fail_on_missing:
asseldonk's avatar
asseldonk committed
570
                                raise Exception('workspace.ini is missing the option "%s" in section [%s] ' % (name, section))
571
                elif fail_on_missing:
asseldonk's avatar
asseldonk committed
572
                    raise Exception('workspace.ini is missing the section [%s]' % section)
573
            return json.dumps({
574
                "content": data,
575
576
                "success": True,
                "mtime": mtime
asseldonk's avatar
asseldonk committed
577
                })
578
579
        except Exception as e:
            return json.dumps({
580
581
582
                "content": "",
                "success": False,
                "error": str(e)
asseldonk's avatar
asseldonk committed
583
                })
584

585
    def set_workspaceini(self, request):
586
        try:
587
588
            request_dict = json.loads(request)
            if not isinstance(request_dict, dict):
asseldonk's avatar
asseldonk committed
589
                raise Exception('Given values to be set in workspace.ini in wrong format')
590
            config = ConfigParser.ConfigParser()
591
            config.read(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF))
592
593
            for section, options in request_dict.iteritems():
                if not isinstance(options, dict):
asseldonk's avatar
asseldonk committed
594
                    raise Exception('Given values to be set in workspace.ini in wrong format')
595
596
597
598
                if not config.has_section(section):
                    config.add_section(section)
                for name, value in options.iteritems():
                    config.set(section, name, value)
599
600
            with open(self.expand(FileSystem.PRIVATE_WORKSPACE_CONF), 'w') as f:
                self._watch_workspaceini()
601
602
                config.write(f)
            return ""
603
        except Exception as e:
604
            return str(e)
asseldonk's avatar
asseldonk committed
605

606
    def _watch_workspaceini(self):
607
        if self.exists(FileSystem.PRIVATE_WORKSPACE_CONF, 'f'):
asseldonk's avatar
asseldonk committed
608
            self.watchservice.subscribe((self._userid, self._workspaceid), FileSystem.PRIVATE_WORKSPACE_CONF)
609
610
611
612
613
614
615
616
617
618
619

class WatchService(object):
    def __init__(self):
        self.subscriber_buffer = []
        self.subscribers = {}
        self.watches = {}
        self.lock = Lock()
        self.monitor = fsmonitor.FSMonitor()
        self.run = True
        self.thread = Thread(target=self._worker)
        self.thread.start()
asseldonk's avatar
asseldonk committed
620
    
621
    def subscribe(self, id, path, pattern=None, reverse=False, hide_hidden=True):
622
623
        if not path:
            return self.unsubscribe(id)
asseldonk's avatar
asseldonk committed
624
        
625
        path = os.path.expanduser(os.path.expandvars(path)).encode('utf8')
asseldonk's avatar
asseldonk committed
626
        
627
628
629
        with self.lock:
            if id not in self.subscribers:
                WatchSubscriber(self, id)
asseldonk's avatar
asseldonk committed
630
            
631
            self.subscribers[id].update(path, pattern, reverse, hide_hidden)
asseldonk's avatar
asseldonk committed
632
    
633
634
635
636
    def unsubscribe(self, id):
        with self.lock:
            if hasattr(id, '__contains__') and None in id:
                for subscriber in self.subscribers.values():
asseldonk's avatar
asseldonk committed
637
                    if False not in map(lambda e,c: c is None or e == c, subscriber.id, id):
638
639
640
                        subscriber.destroy()
            elif id in self.subscribers:
                self.subscribers[id].destroy()
asseldonk's avatar
asseldonk committed
641
    
642
643
    def stop(self):
        self.run = False
asseldonk's avatar
asseldonk committed
644
    
645
646
647
648
649
650
    def _worker(self):
        while self.run:
            events = self.monitor.read_events(0.05)
            if events:
                with self.lock:
                    for event in events:
asseldonk's avatar
asseldonk committed
651
                        if event.action_name in ['delete self','move self']:
652
653
654
                            kind = 'vanish'
                        elif event.action_name == 'modify':
                            kind = 'modify'
asseldonk's avatar
asseldonk committed
655
                        elif event.watch.isdir and event.action_name in ['create','delete','move from','move to']:
656
657
658
659
660
661
                            kind = 'change'
                        else:
                            kind = None
                        if kind:
                            if not event.watch.isdir:
                                if os.path.exists(event.watch.path):
asseldonk's avatar
asseldonk committed
662
                                    event.watch.mtime = os.path.getmtime(event.watch.path)
663
664
665
666
                                else:
                                    event.watch.mtime = -1
                            for subscriber in event.watch.subscribers[:]:
                                subscriber.process(kind, event.name)
asseldonk's avatar
asseldonk committed
667
            
668
669
670
671
            if self.subscriber_buffer:
                with self.lock:
                    for subscriber in self.subscriber_buffer[:]:
                        subscriber.flush(False)
asseldonk's avatar
asseldonk committed
672
        
673
        for subscriber in self.subscribers.items():
674
            subscriber.destroy()
asseldonk's avatar
asseldonk committed
675
        
676
677
        self.monitor.remove_all_watches()
        self.monitor.close()
asseldonk's avatar
asseldonk committed
678
679
    
class WatchSubscriber(object): # this should never be instanced manually
680
    EVENT_DELAYS = {
asseldonk's avatar
asseldonk committed
681
682
        'change': [1.0,0.1],
        'modify': [1.0,0.2]
683
    }
684
685
    MAX_INLINE_SUBJECTS = 10
    MAX_SUBJECT_NAMES = 25
asseldonk's avatar
asseldonk committed
686
    
687
688
689
690
    def __init__(self, service, id):
        if not isinstance(service, WatchService):
            raise TypeError("No valid WatchService instance was provided")
        if id in service.subscribers:
asseldonk's avatar
asseldonk committed
691
            raise RuntimeError("There is already a subscriber with this id: "+str(id))
692
693
694
695
696
697
        self.id = id
        self.service = service
        self.service.subscribers[self.id] = self
        self.watch = None
        self.pattern = None
        self.reverse = None
698
        self.hide_hidden = None
699
        self.event_buffer = {}
700
        self.subject_buffer = {}
asseldonk's avatar
asseldonk committed
701
    
702
703
704
705
706
707
708
709
    def destroy(self):
        self.unbind()
        if self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)
        del self.service.subscribers[self.id]
        del self.service
        del self.watch
        del self.event_buffer
710
        del self.subject_buffer
asseldonk's avatar
asseldonk committed
711
    
712
    def process(self, event, subject=""):
713
714
715
716
717
        if self.watch.isdir and subject:
            if self.hide_hidden and subject.startswith('.'):
                return
            if self.pattern and bool(self.pattern.search(subject)) != self.reverse:
                return
asseldonk's avatar
asseldonk committed
718
            
719
720
721
722
            if event not in self.subject_buffer:
                self.subject_buffer[event] = []
            if subject not in self.subject_buffer[event]:
                self.subject_buffer[event].append(subject)
asseldonk's avatar
asseldonk committed
723
        
724
725
726
        if event in WatchSubscriber.EVENT_DELAYS:
            now = time()
            if event in self.event_buffer:
asseldonk's avatar
asseldonk committed
727
                self.event_buffer[event][1] = now + WatchSubscriber.EVENT_DELAYS[event][1]
728
            else:
asseldonk's avatar
asseldonk committed
729
                self.event_buffer[event] = [now + delay for delay in WatchSubscriber.EVENT_DELAYS[event]] #first & last event
730
731
732
733
                if self not in self.service.subscriber_buffer:
                    self.service.subscriber_buffer.append(self)
        else:
            self.emit(event)
asseldonk's avatar
asseldonk committed
734
    
735
736
737
738
739
740
741
742
743
744
    def flush(self, force=False):
        now = time()
        for event, delays in self.event_buffer.items():
            if force or min(delays) < now:
                self.emit(event)
                del self.event_buffer[event]
        if not self.event_buffer and self in self.service.subscriber_buffer:
            self.service.subscriber_buffer.remove(self)

    def emit(self, event):
asseldonk's avatar
asseldonk committed
745
        if len(self.id) == 3: # window_id, view_id, watch_id
746
747
748
749
750
            data = {
                'event': event,
                'path': self.watch.path,
                'watch_id': self.id[2]
            }
751
752
753
754
755
            if self.watch.isdir:
                if event in self.subject_buffer and self.subject_buffer[event]:
                    subject_count = len(self.subject_buffer[event])
                    data['subject_count'] = subject_count
                    if subject_count <= WatchSubscriber.MAX_INLINE_SUBJECTS:
asseldonk's avatar
asseldonk committed
756
                        data['subject_infos'] = [get_file_info(self.watch.path, subject) for subject in self.subject_buffer[event]]
757
758
759
760
                    elif subject_count <= WatchSubscriber.MAX_SUBJECT_NAMES:
                        data['subject_names'] = self.subject_buffer[event]
                    self.subject_buffer[event] = []
            else:
761
                data['mtime'] = self.watch.mtime
asseldonk's avatar
asseldonk committed
762
            vispa.remote.send_topic("extension.%s.socket.watch" % self.id[1], window_id=self.id[0], data=data)
asseldonk's avatar
asseldonk committed
763
        elif len(self.id) == 2: # userid, workspaceid
764
765
766
767
768
769
            vispa.remote.send_topic('workspace.ini_modified', user_id=self.id[0], data={
                "workspaceId": self.id[1],
                "mtime": self.watch.mtime
            })
        elif hasattr(self.id, '__call__'):
            self.id(event, self)
asseldonk's avatar
asseldonk committed
770
    
771
    def update(self, path, pattern="", reverse=False, hide_hidden=True):
772
773
774
775
776
        self.bind(path)
        if self.watch.isdir and pattern:
            if not self.pattern or self.pattern.pattern != pattern:
                self.pattern = re.compile(pattern)
            self.reverse = reverse
asseldonk's avatar
asseldonk committed
777
778
779
            self.subject_buffer = {event:[
                    subject for subject in subjects if bool(self.pattern.search(subject)) == self.reverse
                ] for event, subjects in self.subject_buffer.items()}
780
781
782
        else:
            self.pattern = None
            self.reverse = None
783
        self.hide_hidden = hide_hidden
asseldonk's avatar
asseldonk committed
784
    
785
786
787
788
789
790
    def bind(self, path):
        if self.watch:
            if self.watch.path == path:
                return
            else:
                self.unbind()
asseldonk's avatar
asseldonk committed
791
        
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
        if path not in self.service.watches:
            if not os.path.exists(path):
                raise IOError("File to be watched does not exist: %s" % path)
            if os.path.isfile(path):
                isdir = False
                watch = self.service.monitor.add_file_watch(path)
            elif os.path.isdir(path):
                isdir = True
                watch = self.service.monitor.add_dir_watch(path)
            else:
                raise IOError("This kind of file can't be watched!")
            watch.isdir = isdir
            watch.subscribers = []
            self.service.watches[path] = watch
        else:
            watch = self.service.watches[path]
asseldonk's avatar
asseldonk committed
808
        
809
810
811
        self.watch = watch
        if self not in watch.subscribers:
            watch.subscribers.append(self)
asseldonk's avatar
asseldonk committed
812
        
813
        self.subject_buffer = {}
asseldonk's avatar
asseldonk committed
814
    
815
816
    def unbind(self):
        if not self.watch:
817
            return
asseldonk's avatar
asseldonk committed
818
        
819
820
821
822
        self.watch.subscribers.remove(self)
        if len(self.watch.subscribers) == 0:
            del self.service.watches[self.watch.path]
            self.service.monitor.remove_watch(self.watch)
asseldonk's avatar
asseldonk committed
823
        
824
        self.watch = None
825

Marcel's avatar
Marcel committed
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
def string_compare(a, b):
    if a == b:
        return 0
    elif a > b:
        return 1
    else:
        return -1

def file_compare(a, b):
    if not a.startswith('.') and not b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and b.startswith('.'):
        return string_compare(a, b)
    elif a.startswith('.') and not b.startswith('.'):
        return 1
    elif not a.startswith('.') and b.startswith('.'):
        return -1